Add broken test for game support issue

Fixes #379

.github/FUNDING.yml

@@ -1,4 +1,5 @@
 # These are supported funding model platforms
 
+liberapay: rubenwardy
 patreon: rubenwardy
 custom: [ "https://rubenwardy.com/donate/" ]

.github/SECURITY.md

@@ -3,7 +3,7 @@
 ## Supported Versions
 
 We only support the latest production version, deployed to <https://content.minetest.net>.
-See the [releases page](https://github.com/minetest/contentdb/releases).
+This is usually the latest `master` commit.
 
 ## Reporting a Vulnerability
 
@@ -12,8 +12,5 @@ to give us time to fix them. You can do that by using one of the methods outline
 
 * https://rubenwardy.com/contact/
 
-Depending on severity, we will either create a private issue for the vulnerability
-and release a security update, or give you permission to file the issue publicly.
-
 For more information on the justification of this policy, see
 [Responsible Disclosure](https://en.wikipedia.org/wiki/Responsible_disclosure).

.github/workflows/test.yml

@@ -6,7 +6,7 @@ jobs:
   test:
     runs-on: ubuntu-latest
     steps:
-    - uses: actions/checkout@v2
+    - uses: actions/checkout@v4
     - name: Copy config
       run: cp utils/ci/* .
     - name: Build the Docker image

README.md

@@ -29,6 +29,9 @@ See [Developer Intro](docs/dev_intro.md) for an overview of the code organisatio
 
 # Create new migration
 ./utils/create_migration.sh
+
+# Delete database
+docker-compose down && sudo rm -rf data/db
 ```
 
 

app/__init__.py

@@ -18,22 +18,65 @@ import datetime
 import os
 import redis
 
-from flask import redirect, url_for, render_template, flash, request, Flask, send_from_directory, make_response
+from flask import redirect, url_for, render_template, flash, request, Flask, send_from_directory, make_response, render_template_string
 from flask_babel import Babel, gettext
 from flask_flatpages import FlatPages
+from flask_flatpages.utils import pygmented_markdown
 from flask_github import GitHub
-from flask_gravatar import Gravatar
 from flask_login import logout_user, current_user, LoginManager
 from flask_mail import Mail
 from flask_wtf.csrf import CSRFProtect
 
 from app.markdown import init_markdown, MARKDOWN_EXTENSIONS, MARKDOWN_EXTENSION_CONFIG
 
+import sentry_sdk
+from sentry_sdk.integrations.flask import FlaskIntegration
+
+
+if os.getenv("SENTRY_DSN"):
+	def before_send(event, hint):
+		from app.tasks import TaskError
+		if "exc_info" in hint:
+			exc_type, exc_value, tb = hint["exc_info"]
+			if isinstance(exc_value, TaskError):
+				return None
+		return event
+
+	environment = os.getenv("SENTRY_ENVIRONMENT")
+	assert environment is not None
+	sentry_sdk.init(
+		dsn=os.getenv("SENTRY_DSN"),
+		environment=environment,
+
+		integrations=[FlaskIntegration()],
+		# Set traces_sample_rate to 1.0 to capture 100%
+		# of transactions for performance monitoring.
+		traces_sample_rate=0.1,
+		# Set profiles_sample_rate to 1.0 to profile 100%
+		# of sampled transactions.
+		# We recommend adjusting this value in production.
+		profiles_sample_rate=0.1,
+
+		before_send=before_send,
+	)
+
+
 app = Flask(__name__, static_folder="public/static")
+
+
+def my_flatpage_renderer(text):
+	# Render with jinja first
+	prerendered_body = render_template_string(text)
+	return pygmented_markdown(prerendered_body, flatpages=pages)
+
+
 app.config["FLATPAGES_ROOT"] = "flatpages"
 app.config["FLATPAGES_EXTENSION"] = ".md"
 app.config["FLATPAGES_MARKDOWN_EXTENSIONS"] = MARKDOWN_EXTENSIONS
 app.config["FLATPAGES_EXTENSION_CONFIG"] = MARKDOWN_EXTENSION_CONFIG
+app.config["FLATPAGES_HTML_RENDERER"] = my_flatpage_renderer
+app.config["WTF_CSRF_TIME_LIMIT"] = None
+
 app.config["BABEL_TRANSLATION_DIRECTORIES"] = "../translations"
 app.config["LANGUAGES"] = {
 	"en": "English",
@@ -50,11 +93,14 @@ app.config["LANGUAGES"] = {
 	"tr": "Türkçe",
 	"uk": "Українська",
 	"vi": "tiếng Việt",
-	"zh_Hans": "汉语",
+	"zh_CN": "汉语",
 }
 
 app.config.from_pyfile(os.environ["FLASK_CONFIG"])
 
+if not app.config["ADMIN_CONTACT_URL"]:
+	raise Exception("Missing config property: ADMIN_CONTACT_URL")
+
 redis_client = redis.Redis.from_url(app.config["REDIS_URL"])
 
 github = GitHub(app)
@@ -62,14 +108,6 @@ csrf = CSRFProtect(app)
 mail = Mail(app)
 pages = FlatPages(app)
 babel = Babel()
-gravatar = Gravatar(app,
-		size=64,
-		rating="g",
-		default="retro",
-		force_default=False,
-		force_lower=False,
-		use_ssl=True,
-		base_url=None)
 init_markdown(app)
 
 login_manager = LoginManager()
@@ -81,11 +119,6 @@ from .sass import init_app as sass
 sass(app)
 
 
-if not app.debug and app.config["MAIL_UTILS_ERROR_SEND_TO"]:
-	from .maillogger import build_handler
-	app.logger.addHandler(build_handler(app))
-
-
 from . import models, template_filters
 
 
@@ -118,7 +151,7 @@ def check_for_ban():
 			if current_user.rank == models.UserRank.BANNED:
 				current_user.rank = models.UserRank.MEMBER
 			models.db.session.commit()
-		elif current_user.ban or current_user.rank == models.UserRank.BANNED:
+		elif current_user.is_banned:
 			if current_user.ban:
 				flash(gettext("Banned:") + " " + current_user.ban.message, "danger")
 			else:
@@ -135,8 +168,7 @@ from .utils import clear_notifications, is_safe_url, create_session
 
 @app.before_request
 def check_for_notifications():
-	if current_user.is_authenticated:
-		clear_notifications(request.path)
+	clear_notifications(request.path)
 
 
 @app.errorhandler(404)
@@ -192,10 +224,23 @@ def set_locale():
 	if locale:
 		expire_date = datetime.datetime.now()
 		expire_date = expire_date + datetime.timedelta(days=5*365)
-		resp.set_cookie("locale", locale, expires=expire_date)
+		resp.set_cookie("locale", locale, expires=expire_date, secure=True, samesite="Lax")
 
 		if current_user.is_authenticated:
 			current_user.locale = locale
 			models.db.session.commit()
 
 	return resp
+
+
+@app.route("/set-nonfree/", methods=["POST"])
+def set_nonfree():
+	resp = redirect(url_for("homepage.home"))
+	if request.cookies.get("hide_nonfree") == "1":
+		resp.set_cookie("hide_nonfree", "0", expires=0, secure=True, samesite="Lax")
+	else:
+		expire_date = datetime.datetime.now()
+		expire_date = expire_date + datetime.timedelta(days=5*365)
+		resp.set_cookie("hide_nonfree", "1", expires=expire_date, secure=True, samesite="Lax")
+
+	return resp

app/_translations.py

@@ -0,0 +1,248 @@
+# THIS FILE IS AUTOGENERATED: utils/extract_translations.py
+
+from flask_babel import gettext
+
+# NOTE: tags: title for server_tools
+pgettext("tags", "Server Moderation and Tools")
+# NOTE: tags: description for server_tools
+pgettext("tags", "Helps with server maintenance and moderation")
+# NOTE: tags: title for pvp
+pgettext("tags", "Player vs Player (PvP)")
+# NOTE: tags: description for pvp
+pgettext("tags", "Designed to be played competitively against other players")
+# NOTE: tags: title for player_effects
+pgettext("tags", "Player Effects / Power Ups")
+# NOTE: tags: description for player_effects
+pgettext("tags", "For content that changes player effects, including physics, for example: speed, jump height or gravity.")
+# NOTE: tags: title for jam_game_2023
+pgettext("tags", "Jam / Game 2023")
+# NOTE: tags: description for jam_game_2023
+pgettext("tags", "Entries to the 2023 Minetest Game Jam ")
+# NOTE: tags: title for mapgen
+pgettext("tags", "Mapgen / Biomes / Decoration")
+# NOTE: tags: description for mapgen
+pgettext("tags", "New mapgen or changes mapgen")
+# NOTE: tags: title for world_tools
+pgettext("tags", "World Maintenance and Tools")
+# NOTE: tags: description for world_tools
+pgettext("tags", "Tools to manage the world")
+# NOTE: tags: title for inventory
+pgettext("tags", "Inventory")
+# NOTE: tags: description for inventory
+pgettext("tags", "Changes the inventory GUI")
+# NOTE: tags: title for developer_tools
+pgettext("tags", "Developer Tools")
+# NOTE: tags: description for developer_tools
+pgettext("tags", "Tools for game and mod developers")
+# NOTE: tags: title for april_fools
+pgettext("tags", "Joke")
+# NOTE: tags: description for april_fools
+pgettext("tags", "For humorous content, meant as a novelty or joke, not to be taken seriously, and that is not meant to be used seriously or long-term.")
+# NOTE: tags: title for singleplayer
+pgettext("tags", "Singleplayer-focused")
+# NOTE: tags: description for singleplayer
+pgettext("tags", "Content that can be played alone")
+# NOTE: tags: title for crafting
+pgettext("tags", "Crafting")
+# NOTE: tags: description for crafting
+pgettext("tags", "Big changes to crafting gameplay")
+# NOTE: tags: title for adventure__rpg
+pgettext("tags", "Adventure / RPG")
+# NOTE: tags: title for shooter
+pgettext("tags", "Shooter")
+# NOTE: tags: description for shooter
+pgettext("tags", "First person shooters (FPS) and more")
+# NOTE: tags: title for sound_music
+pgettext("tags", "Sounds / Music")
+# NOTE: tags: description for sound_music
+pgettext("tags", "Focuses on or adds new sounds or musical things")
+# NOTE: tags: title for tools
+pgettext("tags", "Tools / Weapons / Armor")
+# NOTE: tags: description for tools
+pgettext("tags", "Adds or changes tools, weapons, and armor")
+# NOTE: tags: title for plants_and_farming
+pgettext("tags", "Plants and Farming")
+# NOTE: tags: description for plants_and_farming
+pgettext("tags", "Adds new plants or other farmable resources.")
+# NOTE: tags: title for simulation
+pgettext("tags", "Sims")
+# NOTE: tags: description for simulation
+pgettext("tags", "Mods and games that aim to simulate real life activity. Similar to SimCity/The Sims/OpenTTD/etc.")
+# NOTE: tags: title for chat
+pgettext("tags", "Chat / Commands")
+# NOTE: tags: description for chat
+pgettext("tags", "Focus on player chat/communication or console interaction.")
+# NOTE: tags: title for multiplayer
+pgettext("tags", "Multiplayer-focused")
+# NOTE: tags: description for multiplayer
+pgettext("tags", "Can/should only be used in multiplayer")
+# NOTE: tags: title for complex_installation
+pgettext("tags", "Complex installation")
+# NOTE: tags: description for complex_installation
+pgettext("tags", "Requires futher installation steps, such as installing LuaRocks or editing the trusted mod setting")
+# NOTE: tags: title for magic
+pgettext("tags", "Magic / Enchanting")
+# NOTE: tags: title for building_mechanics
+pgettext("tags", "Building Mechanics and Tools")
+# NOTE: tags: description for building_mechanics
+pgettext("tags", "Adds game mechanics or tools that change how players build.")
+# NOTE: tags: title for oneofakind__original
+pgettext("tags", "One-of-a-kind / Original")
+# NOTE: tags: description for oneofakind__original
+pgettext("tags", "For games and such that are of their own kind, distinct and original in nature to others of the same category.")
+# NOTE: tags: title for building
+pgettext("tags", "Building")
+# NOTE: tags: description for building
+pgettext("tags", "Focuses on building, such as adding new materials or nodes")
+# NOTE: tags: title for decorative
+pgettext("tags", "Decorative")
+# NOTE: tags: description for decorative
+pgettext("tags", "Adds nodes with no other purpose than for use in building")
+# NOTE: tags: title for puzzle
+pgettext("tags", "Puzzle")
+# NOTE: tags: description for puzzle
+pgettext("tags", "Focus on puzzle solving instead of combat")
+# NOTE: tags: title for environment
+pgettext("tags", "Environment / Weather")
+# NOTE: tags: description for environment
+pgettext("tags", "Improves the world, adding weather, ambient sounds, or other environment mechanics")
+# NOTE: tags: title for less_than_px
+pgettext("tags", "<16px")
+# NOTE: tags: description for less_than_px
+pgettext("tags", "Less than 16px")
+# NOTE: tags: title for mobs
+pgettext("tags", "Mobs / Animals / NPCs")
+# NOTE: tags: description for mobs
+pgettext("tags", "Adds mobs, animals, and non-player characters")
+# NOTE: tags: title for custom_mapgen
+pgettext("tags", "Custom mapgen")
+# NOTE: tags: description for custom_mapgen
+pgettext("tags", "Contains a completely custom mapgen implemented in Lua, usually requires worlds to be set to the 'singlenode' mapgen.")
+# NOTE: tags: title for survival
+pgettext("tags", "Survival")
+# NOTE: tags: description for survival
+pgettext("tags", "Written specifically for survival gameplay with a focus on game-balance, difficulty level, or resources available through crafting, mining, ...")
+# NOTE: tags: title for education
+pgettext("tags", "Education")
+# NOTE: tags: description for education
+pgettext("tags", "Either has educational value, or is a tool to help teachers ")
+# NOTE: tags: title for storage
+pgettext("tags", "Storage")
+# NOTE: tags: description for storage
+pgettext("tags", "Adds or improves item storage mechanics")
+# NOTE: tags: title for library
+pgettext("tags", "API / Library")
+# NOTE: tags: description for library
+pgettext("tags", "Primarily adds an API for other mods to use")
+# NOTE: tags: title for hud
+pgettext("tags", "HUD")
+# NOTE: tags: description for hud
+pgettext("tags", "For mods that grant the player extra information in the HUD")
+# NOTE: tags: title for mini-game
+pgettext("tags", "Mini-game")
+# NOTE: tags: description for mini-game
+pgettext("tags", "Adds a mini-game to be played within Minetest")
+# NOTE: tags: title for pve
+pgettext("tags", "Player vs Environment (PvE)")
+# NOTE: tags: description for pve
+pgettext("tags", "For content designed for one or more players that focus on combat against the world, mobs, or NPCs.")
+# NOTE: tags: title for sports
+pgettext("tags", "Sports")
+# NOTE: tags: title for 16px
+pgettext("tags", "16px")
+# NOTE: tags: description for 16px
+pgettext("tags", "For 16px texture packs")
+# NOTE: tags: title for 64px
+pgettext("tags", "64px")
+# NOTE: tags: description for 64px
+pgettext("tags", "For 64px texture packs")
+# NOTE: tags: title for transport
+pgettext("tags", "Transport")
+# NOTE: tags: description for transport
+pgettext("tags", "Adds or changes transportation methods. Includes teleportation, vehicles, ridable mobs, transport infrastructure and thematic content")
+# NOTE: tags: title for seasonal
+pgettext("tags", "Seasonal")
+# NOTE: tags: description for seasonal
+pgettext("tags", "For content generally themed around a certain season or holiday")
+# NOTE: tags: title for food
+pgettext("tags", "Food / Drinks")
+# NOTE: tags: title for commerce
+pgettext("tags", "Commerce / Economy")
+# NOTE: tags: description for commerce
+pgettext("tags", "Related to economies, money, and trading")
+# NOTE: tags: title for gui
+pgettext("tags", "GUI")
+# NOTE: tags: description for gui
+pgettext("tags", "For content whose main utility or features are provided within a GUI, on-screen menu, or similar")
+# NOTE: tags: title for jam_combat_mod
+pgettext("tags", "Jam / Combat 2020")
+# NOTE: tags: description for jam_combat_mod
+pgettext("tags", "For mods created for the Discord \"Combat\" modding event in 2020")
+# NOTE: tags: title for mtg
+pgettext("tags", "Minetest Game improved")
+# NOTE: tags: description for mtg
+pgettext("tags", "Forks of Minetest Game")
+# NOTE: tags: title for jam_weekly_2021
+pgettext("tags", "Jam / Weekly Challenges 2021")
+# NOTE: tags: description for jam_weekly_2021
+pgettext("tags", "For mods created for the Discord \"Weekly Challenges\" modding event in 2021")
+# NOTE: tags: title for creative
+pgettext("tags", "Creative")
+# NOTE: tags: description for creative
+pgettext("tags", "Written specifically or exclusively for use in creative mode. Adds content only available through a creative inventory, or provides tools that facilitate ingame creation and doesn't add difficulty or scarcity")
+# NOTE: tags: title for technology
+pgettext("tags", "Machines / Electronics")
+# NOTE: tags: description for technology
+pgettext("tags", "Adds machines useful in automation, tubes, or power.")
+# NOTE: tags: title for 32px
+pgettext("tags", "32px")
+# NOTE: tags: description for 32px
+pgettext("tags", "For 32px texture packs")
+# NOTE: tags: title for strategy_rts
+pgettext("tags", "Strategy / RTS")
+# NOTE: tags: description for strategy_rts
+pgettext("tags", "Games and mods with a heavy strategy component, whether real-time or turn-based")
+# NOTE: tags: title for skins
+pgettext("tags", "Player customization / Skins")
+# NOTE: tags: description for skins
+pgettext("tags", "Allows the player to customize their character by changing the texture or adding accessories.")
+# NOTE: tags: title for jam_game_2021
+pgettext("tags", "Jam / Game 2021")
+# NOTE: tags: description for jam_game_2021
+pgettext("tags", "Entries to the 2021 Minetest Game Jam")
+# NOTE: tags: title for 128px
+pgettext("tags", "128px+")
+# NOTE: tags: description for 128px
+pgettext("tags", "For 128px or higher texture packs")
+# NOTE: tags: title for jam_game_2022
+pgettext("tags", " Jam / Game 2022")
+# NOTE: tags: description for jam_game_2022
+pgettext("tags", "Entries to the 2022 Minetest Game Jam ")
+# NOTE: content_warnings: title for gore
+pgettext("content_warnings", "Gore")
+# NOTE: content_warnings: description for gore
+pgettext("content_warnings", "Blood, etc")
+# NOTE: content_warnings: title for gambling
+pgettext("content_warnings", "Gambling")
+# NOTE: content_warnings: description for gambling
+pgettext("content_warnings", "Games of chance, gambling games, etc")
+# NOTE: content_warnings: title for violence
+pgettext("content_warnings", "Violence")
+# NOTE: content_warnings: description for violence
+pgettext("content_warnings", "Non-cartoon violence. May be towards fantasy or  human-like characters")
+# NOTE: content_warnings: title for horror
+pgettext("content_warnings", "Fear / Horror")
+# NOTE: content_warnings: description for horror
+pgettext("content_warnings", "Shocking and scary content. May scare young children")
+# NOTE: content_warnings: title for bad_language
+pgettext("content_warnings", "Bad Language")
+# NOTE: content_warnings: description for bad_language
+pgettext("content_warnings", "Contains swearing")
+# NOTE: content_warnings: title for alcohol_tobacco
+pgettext("content_warnings", "Alcohol / Tobacco")
+# NOTE: content_warnings: description for alcohol_tobacco
+pgettext("content_warnings", "Contains alcohol and/or tobacco")
+# NOTE: content_warnings: title for drugs
+pgettext("content_warnings", "Drugs")
+# NOTE: content_warnings: description for drugs
+pgettext("content_warnings", "Contains recreational drugs other than alcohol or tobacco")

app/blueprints/admin/__init__.py

@@ -19,4 +19,4 @@ from flask import Blueprint
 
 bp = Blueprint("admin", __name__)
 
-from . import admin, audit, licenseseditor, tagseditor, versioneditor, warningseditor, email
+from . import admin, audit, licenseseditor, tagseditor, versioneditor, warningseditor, languageseditor, email, approval_stats

app/blueprints/admin/actions.py

@@ -13,7 +13,7 @@
 #
 # You should have received a copy of the GNU Affero General Public License
 # along with this program.  If not, see <https://www.gnu.org/licenses/>.
-
+import datetime
 import os
 from typing import List
 
@@ -23,10 +23,13 @@ from flask import redirect, url_for, flash, current_app
 from sqlalchemy import or_, and_
 
 from app.models import PackageRelease, db, Package, PackageState, PackageScreenshot, MetaPackage, User, \
-	NotificationType, PackageUpdateConfig, License, UserRank, PackageType
+	NotificationType, PackageUpdateConfig, License, UserRank, PackageType, Thread, AuditLogEntry
 from app.tasks.emails import send_pending_digests
 from app.tasks.forumtasks import import_topic_list, check_all_forum_accounts
-from app.tasks.importtasks import import_repo_screenshot, check_zip_release, check_for_updates, update_all_game_support
+from app.tasks.importtasks import import_repo_screenshot, check_zip_release, check_for_updates, update_all_game_support, \
+	import_languages, check_all_zip_files
+from app.tasks.usertasks import import_github_user_ids
+from app.tasks.pkgtasks import notify_about_git_forum_links, clear_removed_packages, check_package_for_broken_links
 from app.utils import add_notification, get_system_user
 
 actions = {}
@@ -51,19 +54,6 @@ def del_stuck_releases():
 	db.session.commit()
 	return redirect(url_for("admin.admin_page"))
 
-
-@action("Import forum topic list")
-def import_topic_list():
-	task = import_topic_list.delay()
-	return redirect(url_for("tasks.check", id=task.id, r=url_for("todo.topics")))
-
-
-@action("Check all forum accounts")
-def check_all_forum_accounts():
-	task = check_all_forum_accounts.delay()
-	return redirect(url_for("tasks.check", id=task.id, r=url_for("admin.admin_page")))
-
-
 @action("Delete unused uploads")
 def clean_uploads():
 	upload_dir = current_app.config['UPLOAD_DIR']
@@ -109,6 +99,29 @@ def del_mod_names():
 	return redirect(url_for("admin.admin_page"))
 
 
+@action("Recalc package scores")
+def recalc_scores():
+	for package in Package.query.all():
+		package.recalculate_score()
+
+	db.session.commit()
+
+	flash("Recalculated package scores", "success")
+	return redirect(url_for("admin.admin_page"))
+
+
+@action("Import forum topic list")
+def do_import_topic_list():
+	task = import_topic_list.delay()
+	return redirect(url_for("tasks.check", id=task.id, r=url_for("admin.admin_page")))
+
+
+@action("Check all forum accounts")
+def check_all_forum_accounts():
+	task = check_all_forum_accounts.delay()
+	return redirect(url_for("tasks.check", id=task.id, r=url_for("admin.admin_page")))
+
+
 @action("Run update configs")
 def run_update_config():
 	check_for_updates.delay()
@@ -121,10 +134,9 @@ def _package_list(packages: List[str]):
 	# Who needs translations?
 	if len(packages) >= 3:
 		packages[len(packages) - 1] = "and " + packages[len(packages) - 1]
-		packages_list = ", ".join(packages)
+		return ", ".join(packages)
 	else:
-		packages_list = " and ".join(packages)
-	return packages_list
+		return " and ".join(packages)
 
 
 @action("Send WIP package notification")
@@ -133,12 +145,12 @@ def remind_wip():
 			Package.state == PackageState.WIP, Package.state == PackageState.CHANGES_NEEDED)))
 	system_user = get_system_user()
 	for user in users:
-		packages = db.session.query(Package.title).filter(
+		packages = Package.query.filter(
 				Package.author_id == user.id,
 				or_(Package.state == PackageState.WIP, Package.state == PackageState.CHANGES_NEEDED)) \
 			.all()
 
-		packages = [pkg[0] for pkg in packages]
+		packages = [pkg.title for pkg in packages]
 		packages_list = _package_list(packages)
 		havent = "haven't" if len(packages) > 1 else "hasn't"
 
@@ -154,12 +166,12 @@ def remind_outdated():
 			Package.update_config.has(PackageUpdateConfig.outdated_at.isnot(None))))
 	system_user = get_system_user()
 	for user in users:
-		packages = db.session.query(Package.title).filter(
+		packages = Package.query.filter(
 				Package.maintainers.contains(user),
 				Package.update_config.has(PackageUpdateConfig.outdated_at.isnot(None))) \
 			.all()
 
-		packages = [pkg[0] for pkg in packages]
+		packages = [pkg.title for pkg in packages]
 		packages_list = _package_list(packages)
 
 		add_notification(user, system_user, NotificationType.PACKAGE_APPROVAL,
@@ -231,15 +243,15 @@ def remind_video_url():
 			and_(Package.video_url == None, Package.type == PackageType.GAME, Package.state == PackageState.APPROVED)))
 	system_user = get_system_user()
 	for user in users:
-		packages = db.session.query(Package.title).filter(
+		packages = Package.query.filter(
 				or_(Package.author == user, Package.maintainers.contains(user)),
 				Package.video_url == None,
 				Package.type == PackageType.GAME,
 				Package.state == PackageState.APPROVED) \
 			.all()
 
-		packages = [pkg[0] for pkg in packages]
-		packages_list = _package_list(packages)
+		package_names = [pkg.title for pkg in packages]
+		packages_list = _package_list(package_names)
 
 		add_notification(user, system_user, NotificationType.PACKAGE_APPROVAL,
 				f"You should add a video to {packages_list}",
@@ -259,7 +271,7 @@ def remind_missing_game_support():
 
 	system_user = get_system_user()
 	for user in users:
-		packages = db.session.query(Package.title).filter(
+		packages = Package.query.filter(
 			Package.maintainers.contains(user),
 			Package.state != PackageState.DELETED,
 			Package.type.in_([PackageType.MOD, PackageType.TXP]),
@@ -267,7 +279,7 @@ def remind_missing_game_support():
 			Package.supports_all_games == False) \
 			.all()
 
-		packages = [pkg[0] for pkg in packages]
+		packages = [pkg.title for pkg in packages]
 		packages_list = _package_list(packages)
 
 		add_notification(user, system_user, NotificationType.PACKAGE_APPROVAL,
@@ -289,17 +301,39 @@ def do_send_pending_digests():
 	send_pending_digests.delay()
 
 
-@action("DANGER: Delete removed packages")
-def del_removed_packages():
-	query = Package.query.filter_by(state=PackageState.DELETED)
-	count = query.count()
-	for pkg in query.all():
-		pkg.review_thread = None
-		db.session.delete(pkg)
-	db.session.commit()
+@action("Import user ids from GitHub")
+def do_import_github_user_ids():
+	task_id = uuid()
+	import_github_user_ids.apply_async((), task_id=task_id)
+	return redirect(url_for("tasks.check", id=task_id, r=url_for("admin.admin_page")))
 
-	flash("Deleted {} soft deleted packages packages".format(count), "success")
-	return redirect(url_for("admin.admin_page"))
+
+@action("Notify about links to git/forums instead of CDB")
+def do_notify_git_forums_links():
+	task_id = uuid()
+	notify_about_git_forum_links.apply_async((), task_id=task_id)
+	return redirect(url_for("tasks.check", id=task_id, r=url_for("admin.admin_page")))
+
+
+@action("Check all zip files")
+def do_check_all_zip_files():
+	task_id = uuid()
+	check_all_zip_files.apply_async((), task_id=task_id)
+	return redirect(url_for("tasks.check", id=task_id, r=url_for("admin.admin_page")))
+
+
+@action("DANGER: Delete less popular removed packages")
+def del_less_popular_removed_packages():
+	task_id = uuid()
+	clear_removed_packages.apply_async((False, ), task_id=task_id)
+	return redirect(url_for("tasks.check", id=task_id, r=url_for("admin.admin_page")))
+
+
+@action("DANGER: Delete all removed packages")
+def del_removed_packages():
+	task_id = uuid()
+	clear_removed_packages.apply_async((True, ), task_id=task_id)
+	return redirect(url_for("tasks.check", id=task_id, r=url_for("admin.admin_page")))
 
 
 @action("DANGER: Check all releases (postReleaseCheckUpdate)")
@@ -322,7 +356,7 @@ def check_releases():
 @action("DANGER: Check latest release of all packages (postReleaseCheckUpdate)")
 def reimport_packages():
 	tasks = []
-	for package in Package.query.filter(Package.state != PackageState.DELETED).all():
+	for package in Package.query.filter(Package.state == PackageState.APPROVED).all():
 		release = package.releases.first()
 		if release:
 			tasks.append(check_zip_release.s(release.id, release.file_path))
@@ -336,6 +370,22 @@ def reimport_packages():
 	return redirect(url_for("todo.view_editor"))
 
 
+@action("DANGER: Import translations")
+def reimport_translations():
+	tasks = []
+	for package in Package.query.filter(Package.state == PackageState.APPROVED).all():
+		release = package.releases.first()
+		if release:
+			tasks.append(import_languages.s(release.id, release.file_path))
+
+	result = group(tasks).apply_async()
+	while not result.ready():
+		import time
+		time.sleep(0.1)
+
+	return redirect(url_for("todo.view_editor"))
+
+
 @action("DANGER: Import screenshots from Git")
 def import_screenshots():
 	packages = Package.query \
@@ -347,3 +397,23 @@ def import_screenshots():
 		import_repo_screenshot.delay(package.id)
 
 	return redirect(url_for("admin.admin_page"))
+
+
+@action("DANGER: Delete empty threads")
+def delete_empty_threads():
+	query = Thread.query.filter(~Thread.replies.any())
+	count = query.count()
+	for thread in query.all():
+		thread.watchers.clear()
+		db.session.delete(thread)
+	db.session.commit()
+
+	flash(f"Deleted {count} threads", "success")
+
+	return redirect(url_for("admin.admin_page"))
+
+
+@action("DANGER: Check for broken links in all packages")
+def check_for_broken_links():
+	for package in Package.query.filter_by(state=PackageState.APPROVED).all():
+		check_package_for_broken_links.delay(package.id)

app/blueprints/admin/admin.py

@@ -19,16 +19,18 @@ from flask_login import current_user, login_user
 from flask_wtf import FlaskForm
 from wtforms import StringField, SubmitField, BooleanField
 from wtforms.validators import InputRequired, Length, Optional
-from app.utils import rank_required, add_audit_log, add_notification, get_system_user, nonempty_or_none
+from app.utils import rank_required, add_audit_log, add_notification, get_system_user, nonempty_or_none, \
+	get_int_or_abort
 from . import bp
 from .actions import actions
 from app.models import UserRank, Package, db, PackageState, User, AuditSeverity, NotificationType, PackageAlias
+from ...querybuilder import QueryBuilder
 
 
 @bp.route("/admin/", methods=["GET", "POST"])
-@rank_required(UserRank.ADMIN)
+@rank_required(UserRank.EDITOR)
 def admin_page():
-	if request.method == "POST":
+	if request.method == "POST" and current_user.rank.at_least(UserRank.ADMIN):
 		action = request.form["action"]
 		if action in actions:
 			ret = actions[action]["func"]()
@@ -38,8 +40,7 @@ def admin_page():
 		else:
 			flash("Unknown action: " + action, "danger")
 
-	deleted_packages = Package.query.filter(Package.state == PackageState.DELETED).all()
-	return render_template("admin/list.html", deleted_packages=deleted_packages, actions=actions)
+	return render_template("admin/list.html", actions=actions)
 
 
 class SwitchUserForm(FlaskForm):
@@ -52,7 +53,7 @@ class SwitchUserForm(FlaskForm):
 def switch_user():
 	form = SwitchUserForm(formdata=request.form)
 	if form.validate_on_submit():
-		user = User.query.filter_by(username=form["username"].data).first()
+		user = User.query.filter_by(username=form.username.data).first()
 		if user is None:
 			flash("Unable to find user", "danger")
 		elif login_user(user):
@@ -179,3 +180,27 @@ def transfer():
 
 	# Process GET or invalid POST
 	return render_template("admin/transfer.html", form=form)
+
+
+@bp.route("/admin/storage/")
+@rank_required(UserRank.EDITOR)
+def storage():
+	qb = QueryBuilder(request.args, cookies=True)
+	qb.only_approved = False
+	packages = qb.build_package_query().all()
+
+	show_all = len(packages) < 100
+	min_size = get_int_or_abort(request.args.get("min_size"), 0 if show_all else 50)
+
+	data = []
+	for package in packages:
+		size_releases = sum([x.file_size_bytes for x in package.releases])
+		size_screenshots = sum([x.file_size_bytes for x in package.screenshots])
+		latest_release = package.releases.first()
+		size_latest = latest_release.file_size_bytes if latest_release else 0
+		size_total = size_releases + size_screenshots
+		if size_total > min_size*1024*1024:
+			data.append([package, size_total, size_releases, size_screenshots, size_latest])
+
+	data.sort(key=lambda x: x[1], reverse=True)
+	return render_template("admin/storage.html", data=data)

app/blueprints/admin/approval_stats.py

@@ -0,0 +1,77 @@
+# ContentDB
+# Copyright (C) 2024 rubenwardy
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program.  If not, see <https://www.gnu.org/licenses/>.
+
+import datetime
+from flask import render_template, request, abort, redirect, url_for, jsonify
+
+from . import bp
+from app.logic.approval_stats import get_approval_statistics
+from app.models import UserRank
+from app.utils import rank_required
+
+
+@bp.route("/admin/approval_stats/")
+@rank_required(UserRank.APPROVER)
+def approval_stats():
+	start = request.args.get("start")
+	end = request.args.get("end")
+	if start and end:
+		try:
+			start = datetime.datetime.fromisoformat(start)
+			end = datetime.datetime.fromisoformat(end)
+		except ValueError:
+			abort(400)
+	elif start:
+		return redirect(url_for("admin.approval_stats", start=start, end=datetime.datetime.utcnow().date().isoformat()))
+	elif end:
+		return redirect(url_for("admin.approval_stats", start="2020-07-01", end=end))
+	else:
+		end = datetime.datetime.utcnow()
+		start = end - datetime.timedelta(days=365)
+
+	stats = get_approval_statistics(start, end)
+	return render_template("admin/approval_stats.html", stats=stats, start=start, end=end)
+
+
+@bp.route("/admin/approval_stats.json")
+@rank_required(UserRank.APPROVER)
+def approval_stats_json():
+	start = request.args.get("start")
+	end = request.args.get("end")
+	if start and end:
+		try:
+			start = datetime.datetime.fromisoformat(start)
+			end = datetime.datetime.fromisoformat(end)
+		except ValueError:
+			abort(400)
+	else:
+		end = datetime.datetime.utcnow()
+		start = end - datetime.timedelta(days=365)
+
+	stats = get_approval_statistics(start, end)
+	for key, value in stats.packages_info.items():
+		stats.packages_info[key] = value.__dict__()
+
+	return jsonify({
+		"start": start.isoformat(),
+		"end": end.isoformat(),
+		"editor_approvals": stats.editor_approvals,
+		"packages_info": stats.packages_info,
+		"turnaround_time": {
+			"avg": stats.avg_turnaround_time,
+			"max": stats.max_turnaround_time,
+		},
+	})

app/blueprints/admin/audit.py

@@ -37,6 +37,10 @@ def audit():
 			abort(404)
 		query = query.filter_by(causer=user)
 
+	if "q" in request.args:
+		q = request.args["q"]
+		query = query.filter(AuditLogEntry.title.ilike(f"%{q}%"))
+
 	pagination = query.paginate(page=page, per_page=num)
 	return render_template("admin/audit.html", log=pagination.items, pagination=pagination)
 

app/blueprints/admin/email.py

@@ -22,14 +22,14 @@ from wtforms.validators import InputRequired, Length
 
 from app.markdown import render_markdown
 from app.tasks.emails import send_user_email, send_bulk_email as task_send_bulk
-from app.utils import rank_required, add_audit_log
+from app.utils import rank_required, add_audit_log, normalize_line_endings
 from . import bp
 from app.models import UserRank, User, AuditSeverity
 
 
 class SendEmailForm(FlaskForm):
 	subject = StringField("Subject", [InputRequired(), Length(1, 300)])
-	text    = TextAreaField("Message", [InputRequired()])
+	text    = TextAreaField("Message", [InputRequired()], filters=[normalize_line_endings])
 	submit  = SubmitField("Send")
 
 

app/blueprints/admin/languageseditor.py

@@ -0,0 +1,73 @@
+# ContentDB
+# Copyright (C) 2018-24 rubenwardy
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program.  If not, see <https://www.gnu.org/licenses/>.
+
+
+from flask import redirect, render_template, abort, url_for
+from flask_login import current_user
+from flask_wtf import FlaskForm
+from wtforms import StringField, TextAreaField, SubmitField
+from wtforms.validators import InputRequired, Length, Optional
+
+from app.models import db, AuditSeverity, UserRank, Language, Package, PackageState, PackageTranslation
+from app.utils import add_audit_log, rank_required, normalize_line_endings
+from . import bp
+
+
+@bp.route("/admin/languages/")
+@rank_required(UserRank.ADMIN)
+def language_list():
+	at_least_one_count = db.session.query(PackageTranslation.package_id).group_by(PackageTranslation.package_id).count()
+	total_package_count = Package.query.filter_by(state=PackageState.APPROVED).count()
+	return render_template("admin/languages/list.html",
+		languages=Language.query.all(), total_package_count=total_package_count,
+		at_least_one_count=at_least_one_count)
+
+
+class LanguageForm(FlaskForm):
+	id = StringField("Id", [InputRequired(), Length(2, 10)])
+	title = TextAreaField("Title", [Optional(), Length(2, 100)], filters=[normalize_line_endings])
+	submit = SubmitField("Save")
+
+
+@bp.route("/admin/languages/new/", methods=["GET", "POST"])
+@bp.route("/admin/languages/<id_>/edit/", methods=["GET", "POST"])
+@rank_required(UserRank.ADMIN)
+def create_edit_language(id_=None):
+	language = None
+	if id_ is not None:
+		language = Language.query.filter_by(id=id_).first()
+		if language is None:
+			abort(404)
+
+	form = LanguageForm(obj=language)
+	if form.validate_on_submit():
+		if language is None:
+			language = Language()
+			db.session.add(language)
+			form.populate_obj(language)
+
+			add_audit_log(AuditSeverity.EDITOR, current_user, f"Created language {language.id}",
+					url_for("admin.create_edit_language", id_=language.id))
+		else:
+			form.populate_obj(language)
+
+			add_audit_log(AuditSeverity.EDITOR, current_user, f"Edited language {language.id}",
+					url_for("admin.create_edit_language", id_=language.id))
+
+		db.session.commit()
+		return redirect(url_for("admin.create_edit_language", id_=language.id))
+
+	return render_template("admin/languages/edit.html", language=language, form=form)

app/blueprints/admin/tagseditor.py

@@ -23,7 +23,7 @@ from wtforms.validators import InputRequired, Length, Optional, Regexp
 
 from . import bp
 from app.models import Permission, Tag, db, AuditSeverity
-from app.utils import add_audit_log
+from app.utils import add_audit_log, normalize_line_endings
 
 
 @bp.route("/tags/")
@@ -44,7 +44,7 @@ def tag_list():
 
 class TagForm(FlaskForm):
 	title = StringField("Title", [InputRequired(), Length(3, 100)])
-	description = TextAreaField("Description", [Optional(), Length(0, 500)])
+	description = TextAreaField("Description", [Optional(), Length(0, 500)], filters=[normalize_line_endings])
 	name = StringField("Name", [Optional(), Length(1, 20), Regexp("^[a-z0-9_]", 0,
 			"Lower case letters (a-z), digits (0-9), and underscores (_) only")])
 	submit = SubmitField("Save")

app/blueprints/admin/warningseditor.py

@@ -20,7 +20,7 @@ from flask_wtf import FlaskForm
 from wtforms import StringField, TextAreaField, SubmitField
 from wtforms.validators import InputRequired, Length, Optional, Regexp
 
-from app.utils import rank_required
+from app.utils import rank_required, normalize_line_endings
 from . import bp
 from app.models import UserRank, ContentWarning, db
 
@@ -33,7 +33,7 @@ def warning_list():
 
 class WarningForm(FlaskForm):
 	title = StringField("Title", [InputRequired(), Length(3, 100)])
-	description = TextAreaField("Description", [Optional(), Length(0, 500)])
+	description = TextAreaField("Description", [Optional(), Length(0, 500)], filters=[normalize_line_endings])
 	name = StringField("Name", [Optional(), Length(1, 20),
 					Regexp("^[a-z0-9_]", 0, "Lower case letters (a-z), digits (0-9), and underscores (_) only")])
 	submit = SubmitField("Save")

app/blueprints/api/__init__.py

@@ -14,8 +14,36 @@
 # You should have received a copy of the GNU Affero General Public License
 # along with this program.  If not, see <https://www.gnu.org/licenses/>.
 
+import json
 from flask import Blueprint
 
+from .support import error
+
 bp = Blueprint("api", __name__)
 
+
 from . import tokens, endpoints
+
+
+@bp.errorhandler(400)
+@bp.errorhandler(401)
+@bp.errorhandler(403)
+@bp.errorhandler(404)
+def handle_exception(e):
+	"""Return JSON instead of HTML for HTTP errors."""
+	# start with the correct headers and status code from the error
+	response = e.get_response()
+	# replace the body with JSON
+	response.data = json.dumps({
+		"success": False,
+		"code": e.code,
+		"name": e.name,
+		"description": e.description,
+	})
+	response.content_type = "application/json"
+	return response
+
+
+@bp.route("/api/<path:path>")
+def page_not_found(path):
+	error(404, "Endpoint or method not found")

app/blueprints/api/auth.py

@@ -39,7 +39,7 @@ def is_api_authd(f):
 			if token is None:
 				error(403, "Unknown API token")
 		else:
-			abort(403, "Unsupported authentication method")
+			error(403, "Unsupported authentication method")
 
 		return f(token=token, *args, **kwargs)
 

app/blueprints/api/endpoints.py

@@ -15,12 +15,12 @@
 # along with this program.  If not, see <https://www.gnu.org/licenses/>.
 
 import math
-from functools import wraps
+import os
 from typing import List
 
 import flask_sqlalchemy
-from flask import request, jsonify, current_app, Response
-from flask_login import current_user, login_required
+from flask import request, jsonify, current_app
+from flask_babel import gettext
 from sqlalchemy import and_, or_
 from sqlalchemy.orm import joinedload
 from sqlalchemy.sql.expression import func
@@ -28,52 +28,35 @@ from sqlalchemy.sql.expression import func
 from app import csrf
 from app.logic.graphs import get_package_stats, get_package_stats_for_user, get_all_package_stats
 from app.markdown import render_markdown
-from app.models import Tag, PackageState, PackageType, Package, db, PackageRelease, Permission, ForumTopic, \
+from app.models import Tag, PackageState, PackageType, Package, db, PackageRelease, Permission, \
 	MinetestRelease, APIToken, PackageScreenshot, License, ContentWarning, User, PackageReview, Thread, Collection, \
-	PackageAlias
+	PackageAlias, Language
 from app.querybuilder import QueryBuilder
-from app.utils import is_package_page, get_int_or_abort, url_set_query, abs_url, is_yes, get_request_date
+from app.utils import is_package_page, get_int_or_abort, url_set_query, abs_url, is_yes, get_request_date, cached, \
+	cors_allowed
+from app.utils.minetest_hypertext import html_to_minetest, package_info_as_hypertext, package_reviews_as_hypertext
 from . import bp
 from .auth import is_api_authd
 from .support import error, api_create_vcs_release, api_create_zip_release, api_create_screenshot, \
 	api_order_screenshots, api_edit_package, api_set_cover_image
-from app.utils.minetest_hypertext import html_to_minetest
-
-
-def cors_allowed(f):
-	@wraps(f)
-	def inner(*args, **kwargs):
-		res: Response = f(*args, **kwargs)
-		res.headers["Access-Control-Allow-Origin"] = "*"
-		res.headers["Access-Control-Allow-Methods"] = "GET, POST, PUT, DELETE, OPTIONS"
-		res.headers["Access-Control-Allow-Headers"] = "Content-Type, Authorization"
-		return res
-	return inner
-
-
-def cached(max_age: int):
-	def decorator(f):
-		@wraps(f)
-		def inner(*args, **kwargs):
-			res: Response = f(*args, **kwargs)
-			res.cache_control.max_age = max_age
-			return res
-		return inner
-
-	return decorator
 
 
 @bp.route("/api/packages/")
 @cors_allowed
 @cached(300)
 def packages():
-	qb = QueryBuilder(request.args)
+	allowed_languages = set([x[0] for x in db.session.query(Language.id).all()])
+	lang = request.accept_languages.best_match(allowed_languages)
+
+	qb = QueryBuilder(request.args, lang=lang)
 	query = qb.build_package_query()
 
-	if request.args.get("fmt") == "keys":
+	fmt = request.args.get("fmt")
+	if fmt == "keys":
 		return jsonify([pkg.as_key_dict() for pkg in query.all()])
 
-	pkgs = qb.convert_to_dictionary(query.all())
+	include_vcs = fmt == "vcs"
+	pkgs = qb.convert_to_dictionary(query.all(), include_vcs)
 	if "engine_version" in request.args or "protocol_version" in request.args:
 		pkgs = [pkg for pkg in pkgs if pkg.get("release")]
 
@@ -84,32 +67,92 @@ def packages():
 			"limit" not in request.args:
 		featured_lut = set()
 		featured = qb.convert_to_dictionary(query.filter(
-			Package.collections.any(and_(Collection.name == "featured", Collection.author.has(username="ContentDB")))).all())
+			Package.collections.any(and_(Collection.name == "featured", Collection.author.has(username="ContentDB")))).all(),
+			include_vcs)
 		for pkg in featured:
 			featured_lut.add(f"{pkg['author']}/{pkg['name']}")
-			pkg["short_description"] = "Featured. " + pkg["short_description"]
+			pkg["short_description"] = gettext("Featured") + ". " + pkg["short_description"]
+			pkg["featured"] = True
 
 		not_featured = [pkg for pkg in pkgs if f"{pkg['author']}/{pkg['name']}" not in featured_lut]
 		pkgs = featured + not_featured
 
-	return jsonify(pkgs)
+	resp = jsonify(pkgs)
+	resp.vary = "Accept-Language"
+	return resp
 
 
 @bp.route("/api/packages/<author>/<name>/")
 @is_package_page
 @cors_allowed
 def package_view(package):
-	return jsonify(package.as_dict(current_app.config["BASE_URL"]))
+	allowed_languages = set([x[0] for x in db.session.query(Language.id).all()])
+	lang = request.accept_languages.best_match(allowed_languages)
+
+	data = package.as_dict(current_app.config["BASE_URL"], lang=lang)
+	resp = jsonify(data)
+	resp.vary = "Accept-Language"
+	return resp
+
+
+@bp.route("/api/packages/<author>/<name>/for-client/")
+@is_package_page
+@cors_allowed
+def package_view_client(package: Package):
+	protocol_version = request.args.get("protocol_version")
+	engine_version = request.args.get("engine_version")
+	if protocol_version or engine_version:
+		version = MinetestRelease.get(engine_version, get_int_or_abort(protocol_version))
+	else:
+		version = None
+
+	allowed_languages = set([x[0] for x in db.session.query(Language.id).all()])
+	lang = request.accept_languages.best_match(allowed_languages)
+
+	data = package.as_dict(current_app.config["BASE_URL"], version, lang=lang, screenshots_dict=True)
+
+	formspec_version = get_int_or_abort(request.args["formspec_version"])
+	include_images = is_yes(request.args.get("include_images", "true"))
+	html = render_markdown(data["long_description"])
+	page_url = package.get_url("packages.view", absolute=True)
+	data["long_description"] = html_to_minetest(html, page_url, formspec_version, include_images)
+
+	data["info_hypertext"] = package_info_as_hypertext(package, formspec_version)
+
+	data["download_size"] = package.get_download_release(version).file_size
+
+	data["reviews"] = {
+		"positive": package.reviews.filter(PackageReview.rating > 3).count(),
+		"neutral": package.reviews.filter(PackageReview.rating == 3).count(),
+		"negative": package.reviews.filter(PackageReview.rating < 3).count(),
+	}
+
+	resp = jsonify(data)
+	resp.vary = "Accept-Language"
+	return resp
+
+
+@bp.route("/api/packages/<author>/<name>/for-client/reviews/")
+@is_package_page
+@cors_allowed
+def package_view_client_reviews(package: Package):
+	formspec_version = get_int_or_abort(request.args["formspec_version"])
+	data = package_reviews_as_hypertext(package, formspec_version)
+
+	resp = jsonify(data)
+	resp.vary = "Accept-Language"
+	return resp
 
 
 @bp.route("/api/packages/<author>/<name>/hypertext/")
 @is_package_page
 @cors_allowed
 def package_hypertext(package):
-	formspec_version = request.args["formspec_version"]
+	formspec_version = get_int_or_abort(request.args["formspec_version"])
 	include_images = is_yes(request.args.get("include_images", "true"))
 	html = render_markdown(package.desc)
-	return jsonify(html_to_minetest(html, formspec_version, include_images))
+	page_url = package.get_url("packages.view", absolute=True)
+	return jsonify(html_to_minetest(html, page_url, formspec_version, include_images))
 
 
 @bp.route("/api/packages/<author>/<name>/", methods=["PUT"])
@@ -167,6 +210,7 @@ def resolve_package_deps(out, package, only_hard, depth=1):
 @bp.route("/api/packages/<author>/<name>/dependencies/")
 @is_package_page
 @cors_allowed
+@cached(300)
 def package_dependencies(package):
 	only_hard = request.args.get("only_hard")
 
@@ -184,24 +228,6 @@ def topics():
 	return jsonify([t.as_dict() for t in query.all()])
 
 
-@bp.route("/api/topic_discard/", methods=["POST"])
-@login_required
-def topic_set_discard():
-	tid = request.args.get("tid")
-	discard = request.args.get("discard")
-	if tid is None or discard is None:
-		error(400, "Missing topic ID or discard bool")
-
-	topic = ForumTopic.query.get(tid)
-	if not topic.check_perm(current_user, Permission.TOPIC_DISCARD):
-		error(403, "Permission denied, need: TOPIC_DISCARD")
-
-	topic.discarded = discard == "true"
-	db.session.commit()
-
-	return jsonify(topic.as_dict())
-
-
 @bp.route("/api/whoami/")
 @is_api_authd
 @cors_allowed
@@ -237,7 +263,7 @@ def markdown():
 def list_all_releases():
 	query = PackageRelease.query.filter_by(approved=True) \
 			.filter(PackageRelease.package.has(state=PackageState.APPROVED)) \
-			.order_by(db.desc(PackageRelease.releaseDate))
+			.order_by(db.desc(PackageRelease.created_at))
 
 	if "author" in request.args:
 		author = User.query.filter_by(username=request.args["author"]).first()
@@ -279,15 +305,19 @@ def create_release(token, package):
 	else:
 		data = request.form
 
-	if "title" not in data:
-		error(400, "Title is required in the POST data")
+	if not ("title" in data or "name" in data):
+		error(400, "name is required in the POST data")
+
+	name = data.get("name")
+	title = data.get("title") or name
+	name = name or title
 
 	if data.get("method") == "git":
 		for option in ["method", "ref"]:
 			if option not in data:
 				error(400, option + " is required in the POST data")
 
-		return api_create_vcs_release(token, package, data["title"], data["ref"])
+		return api_create_vcs_release(token, package, name, title, data.get("release_notes"), data["ref"])
 
 	elif request.files:
 		file = request.files.get("file")
@@ -296,7 +326,7 @@ def create_release(token, package):
 
 		commit_hash = data.get("commit")
 
-		return api_create_zip_release(token, package, data["title"], file, None, None, "API", commit_hash)
+		return api_create_zip_release(token, package,  name, title, data.get("release_notes"), file, None, None, "API", commit_hash)
 
 	else:
 		error(400, "Unknown release-creation method. Specify the method or provide a file.")
@@ -335,6 +365,9 @@ def delete_release(token: APIToken, package: Package, id: int):
 	db.session.delete(release)
 	db.session.commit()
 
+	if release.file_path and os.path.isfile(release.file_path):
+		os.remove(release.file_path)
+
 	return jsonify({"success": True})
 
 
@@ -406,6 +439,8 @@ def delete_screenshot(token: APIToken, package: Package, id: int):
 	db.session.delete(ss)
 	db.session.commit()
 
+	os.remove(ss.file_path)
+
 	return jsonify({ "success": True })
 
 
@@ -521,7 +556,7 @@ def all_package_stats():
 
 @bp.route("/api/scores/")
 @cors_allowed
-@cached(300)
+@cached(900)
 def package_scores():
 	qb = QueryBuilder(request.args)
 	query = qb.build_package_query()
@@ -532,18 +567,21 @@ def package_scores():
 
 @bp.route("/api/tags/")
 @cors_allowed
+@cached(60*60)
 def tags():
 	return jsonify([tag.as_dict() for tag in Tag.query.all() ])
 
 
 @bp.route("/api/content_warnings/")
 @cors_allowed
+@cached(60*60)
 def content_warnings():
 	return jsonify([warning.as_dict() for warning in ContentWarning.query.all() ])
 
 
 @bp.route("/api/licenses/")
 @cors_allowed
+@cached(60*60)
 def licenses():
 	all_licenses = License.query.order_by(db.asc(License.name)).all()
 	return jsonify([{"name": license.name, "is_foss": license.is_foss} for license in all_licenses])
@@ -551,6 +589,7 @@ def licenses():
 
 @bp.route("/api/homepage/")
 @cors_allowed
+@cached(300)
 def homepage():
 	query = Package.query.filter_by(state=PackageState.APPROVED)
 	count = query.count()
@@ -567,7 +606,7 @@ def homepage():
 
 	updated = db.session.query(Package).select_from(PackageRelease).join(Package) \
 			.filter_by(state=PackageState.APPROVED) \
-			.order_by(db.desc(PackageRelease.releaseDate)) \
+			.order_by(db.desc(PackageRelease.created_at)) \
 			.limit(20).all()
 	updated = updated[:4]
 
@@ -624,6 +663,12 @@ def versions():
 			for rel in MinetestRelease.query.all() if rel.get_actual() is not None])
 
 
+@bp.route("/api/languages/")
+@cors_allowed
+def languages():
+	return jsonify([x.as_dict() for x in Language.query.all()])
+
+
 @bp.route("/api/dependencies/")
 @cors_allowed
 def all_deps():
@@ -668,6 +713,7 @@ def user_view(username: str):
 
 @bp.route("/api/users/<username>/stats/")
 @cors_allowed
+@cached(300)
 def user_stats(username: str):
 	user = User.query.filter_by(username=username).first()
 	if user is None:
@@ -680,6 +726,7 @@ def user_stats(username: str):
 
 @bp.route("/api/cdb_schema/")
 @cors_allowed
+@cached(60*60)
 def json_schema():
 	tags = Tag.query.all()
 	warnings = ContentWarning.query.all()
@@ -785,6 +832,11 @@ def json_schema():
 				"type": ["string", "null"],
 				"format": "uri"
 			},
+			"translation_url": {
+				"description": "URL to send users interested in translating your package",
+				"type": ["string", "null"],
+				"format": "uri"
+			}
 		},
 	})
 
@@ -793,14 +845,14 @@ def json_schema():
 @csrf.exempt
 @cors_allowed
 def hypertext():
-	formspec_version = request.args["formspec_version"]
+	formspec_version = get_int_or_abort(request.args["formspec_version"])
 	include_images = is_yes(request.args.get("include_images", "true"))
 
 	html = request.data.decode("utf-8")
 	if request.content_type == "text/markdown":
 		html = render_markdown(html)
 
-	return jsonify(html_to_minetest(html, formspec_version, include_images))
+	return jsonify(html_to_minetest(html, "", formspec_version, include_images))
 
 
 @bp.route("/api/collections/")
@@ -825,18 +877,21 @@ def collection_list():
 
 
 @bp.route("/api/collections/<author>/<name>/")
+@is_api_authd
 @cors_allowed
-def collection_view(author, name):
+def collection_view(token, author, name):
+	user = token.owner if token else None
+
 	collection = Collection.query \
 		.filter(Collection.name == name, Collection.author.has(username=author)) \
 		.one_or_404()
 
-	if not collection.check_perm(current_user, Permission.VIEW_COLLECTION):
+	if not collection.check_perm(user, Permission.VIEW_COLLECTION):
 		error(404, "Collection not found")
 
 	items = collection.items
-	if collection.check_perm(current_user, Permission.EDIT_COLLECTION):
-		items = [x for x in items if x.package.check_perm(current_user, Permission.VIEW_PACKAGE)]
+	if not collection.check_perm(user, Permission.EDIT_COLLECTION):
+		items = [x for x in items if x.package.check_perm(user, Permission.VIEW_PACKAGE)]
 
 	ret = collection.as_dict()
 	ret["items"] = [x.as_dict() for x in items]
@@ -844,6 +899,8 @@ def collection_view(author, name):
 
 
 @bp.route("/api/updates/")
+@cors_allowed
+@cached(300)
 def updates():
 	protocol_version = get_int_or_abort(request.args.get("protocol_version"))
 	minetest_version = request.args.get("engine_version")

app/blueprints/api/support.py

@@ -14,7 +14,7 @@
 # You should have received a copy of the GNU Affero General Public License
 # along with this program.  If not, see <https://www.gnu.org/licenses/>.
 
-
+from typing import Optional
 from flask import jsonify, abort, make_response, url_for, current_app
 
 from app.logic.packages import do_edit_package
@@ -38,14 +38,14 @@ def guard(f):
 	return ret
 
 
-def api_create_vcs_release(token: APIToken, package: Package, title: str, ref: str,
+def api_create_vcs_release(token: APIToken, package: Package, name: str, title: Optional[str], release_notes: Optional[str], ref: str,
 		min_v: MinetestRelease = None, max_v: MinetestRelease = None, reason="API"):
 	if not token.can_operate_on_package(package):
 		error(403, "API token does not have access to the package")
 
 	reason += ", token=" + token.name
 
-	rel = guard(do_create_vcs_release)(token.owner, package, title, ref, min_v, max_v, reason)
+	rel = guard(do_create_vcs_release)(token.owner, package, name, title, release_notes, ref, min_v, max_v, reason)
 
 	return jsonify({
 		"success": True,
@@ -54,14 +54,14 @@ def api_create_vcs_release(token: APIToken, package: Package, title: str, ref: s
 	})
 
 
-def api_create_zip_release(token: APIToken, package: Package, title: str, file,
+def api_create_zip_release(token: APIToken, package: Package, name: str, title: Optional[str], release_notes: Optional[str], file,
 		min_v: MinetestRelease = None, max_v: MinetestRelease = None, reason="API", commit_hash: str = None):
 	if not token.can_operate_on_package(package):
 		error(403, "API token does not have access to the package")
 
 	reason += ", token=" + token.name
 
-	rel = guard(do_create_zip_release)(token.owner, package, title, file, min_v, max_v, reason, commit_hash)
+	rel = guard(do_create_zip_release)(token.owner, package, name, title, release_notes, file, min_v, max_v, reason, commit_hash)
 
 	return jsonify({
 		"success": True,

app/blueprints/api/tokens.py

@@ -59,10 +59,7 @@ def list_tokens(username):
 @bp.route("/users/<username>/tokens/<int:id>/edit/", methods=["GET", "POST"])
 @login_required
 def create_edit_token(username, id=None):
-	user = User.query.filter_by(username=username).first()
-	if user is None:
-		abort(404)
-
+	user = User.query.filter_by(username=username).one_or_404()
 	if not user.check_perm(current_user, Permission.CREATE_TOKEN):
 		abort(403)
 

app/blueprints/collections/__init__.py

@@ -25,7 +25,7 @@ from wtforms import StringField, BooleanField, SubmitField, FieldList, HiddenFie
 from wtforms.validators import InputRequired, Length, Optional, Regexp
 
 from app.models import Collection, db, Package, Permission, CollectionPackage, User, UserRank, AuditSeverity
-from app.utils import nonempty_or_none
+from app.utils import nonempty_or_none, normalize_line_endings
 from app.utils.models import is_package_page, add_audit_log, create_session
 
 bp = Blueprint("collections", __name__)
@@ -67,7 +67,7 @@ def view(author, name):
 		abort(404)
 
 	items = collection.items
-	if collection.check_perm(current_user, Permission.EDIT_COLLECTION):
+	if not collection.check_perm(current_user, Permission.EDIT_COLLECTION):
 		items = [x for x in items if x.package.check_perm(current_user, Permission.VIEW_PACKAGE)]
 
 	return render_template("collections/view.html", collection=collection, items=items)
@@ -78,8 +78,9 @@ class CollectionForm(FlaskForm):
 	name = StringField("URL", [Optional(), Length(1, 20), Regexp("^[a-z0-9_]", 0,
 		"Lower case letters (a-z), digits (0-9), and underscores (_) only")])
 	short_description = StringField(lazy_gettext("Short Description"), [Optional(), Length(0, 200)])
-	long_description = TextAreaField(lazy_gettext("Page Content"), [Optional()], filters=[nonempty_or_none])
+	long_description = TextAreaField(lazy_gettext("Page Content"), [Optional()], filters=[nonempty_or_none, normalize_line_endings])
 	private = BooleanField(lazy_gettext("Private"))
+	pinned = BooleanField(lazy_gettext("Pinned to my profile"))
 	descriptions = FieldList(
 		StringField(lazy_gettext("Short Description"), [Optional(), Length(0, 500)], filters=[nonempty_or_none]),
 		min_entries=0)
@@ -122,6 +123,7 @@ def create_edit(author=None, name=None):
 	if request.method == "GET":
 		# HACK: fix bug in wtforms
 		form.private.data = collection.private if collection else False
+		form.pinned.data = collection.pinned if collection else False
 		if collection:
 			for item in collection.items:
 				form.descriptions.append_entry(item.description)
@@ -129,6 +131,7 @@ def create_edit(author=None, name=None):
 				form.package_removed.append_entry("0")
 		else:
 			form.name = None
+			form.pinned = None
 
 	if form.validate_on_submit():
 		ret = handle_create_edit(collection, form, initial_packages, author)
@@ -319,6 +322,7 @@ def package_add(package):
 @login_required
 def package_toggle_favorite(package):
 	collection, _is_new = get_or_create_favorites(db.session)
+	collection.author = current_user
 
 	if toggle_package(collection, package):
 		msg = gettext("Added package to favorites collection")

app/blueprints/feeds/__init__.py

@@ -0,0 +1,179 @@
+# ContentDB
+# Copyright (C) 2024 rubenwardy
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program.  If not, see <https://www.gnu.org/licenses/>.
+
+
+from flask import Blueprint, jsonify, render_template, make_response
+from flask_babel import gettext
+
+from app.markdown import render_markdown
+from app.models import Package, PackageState, db, PackageRelease
+from app.utils import is_package_page, abs_url_for, cached, cors_allowed
+
+bp = Blueprint("feeds", __name__)
+
+
+def _make_feed(title: str, feed_url: str, items: list):
+	return {
+		"version": "https://jsonfeed.org/version/1",
+		"title": title,
+		"description": gettext("Welcome to the best place to find Minetest mods, games, and texture packs"),
+		"home_page_url": "https://content.minetest.net/",
+		"feed_url": feed_url,
+		"icon": "https://content.minetest.net/favicon-128.png",
+		"expired": False,
+		"items": items,
+	}
+
+
+def _render_link(url: str):
+	return f"<p><a href='{url}'>Read more</a></p>"
+
+
+def _get_new_packages_feed(feed_url: str) -> dict:
+	packages = (Package.query
+		.filter(Package.state == PackageState.APPROVED)
+		.order_by(db.desc(Package.approved_at))
+		.limit(100)
+		.all())
+
+	items = [{
+			"id": package.get_url("packages.view", absolute=True),
+			"language": "en",
+			"title": f"New: {package.title}",
+			"content_html": render_markdown(package.desc) \
+				if package.desc else _render_link(package.get_url("packages.view", absolute=True)),
+			"author": {
+				"name": package.author.display_name,
+				"avatar": package.author.get_profile_pic_url(absolute=True),
+				"url": abs_url_for("users.profile", username=package.author.username),
+			},
+			"image": package.get_thumb_url(level=4, abs=True, format="png"),
+			"url": package.get_url("packages.view", absolute=True),
+			"summary": package.short_desc,
+			"date_published": package.approved_at.isoformat(timespec="seconds") + "Z",
+			"tags": ["new_package"],
+		} for package in packages]
+
+	return _make_feed(gettext("ContentDB new packages"), feed_url, items)
+
+
+def _get_releases_feed(query, feed_url: str):
+	releases = (query
+		.filter(PackageRelease.package.has(state=PackageState.APPROVED), PackageRelease.approved==True)
+		.order_by(db.desc(PackageRelease.created_at))
+		.limit(250)
+		.all())
+
+	items = [{
+			"id": release.package.get_url("packages.view_release", id=release.id, absolute=True),
+			"language": "en",
+			"title": f"\"{release.package.title}\" updated: {release.title}",
+			"content_html": render_markdown(release.release_notes) \
+				if release.release_notes else _render_link(release.package.get_url("packages.view_release", id=release.id, absolute=True)),
+			"author": {
+				"name": release.package.author.display_name,
+				"avatar": release.package.author.get_profile_pic_url(absolute=True),
+				"url": abs_url_for("users.profile", username=release.package.author.username),
+			},
+			"url": release.package.get_url("packages.view_release", id=release.id, absolute=True),
+			"image": release.package.get_thumb_url(level=4, abs=True, format="png"),
+			"summary": release.summary,
+			"date_published": release.created_at.isoformat(timespec="seconds") + "Z",
+			"tags": ["release"],
+		} for release in releases]
+
+	return _make_feed(gettext("ContentDB package updates"), feed_url, items)
+
+
+def _get_all_feed(feed_url: str):
+	releases = _get_releases_feed(PackageRelease.query, "")["items"]
+	packages = _get_new_packages_feed("")["items"]
+	items = releases + packages
+	items.sort(reverse=True, key=lambda x: x["date_published"])
+
+	return _make_feed(gettext("ContentDB all"), feed_url, items)
+
+
+def _atomify(feed):
+	resp = make_response(render_template("feeds/json_to_atom.xml", feed=feed))
+	resp.headers["Content-type"] = "application/atom+xml; charset=utf-8"
+	return resp
+
+
+@bp.route("/feeds/all.json")
+@cors_allowed
+@cached(1800)
+def all_json():
+	feed = _get_all_feed(abs_url_for("feeds.all_json"))
+	return jsonify(feed)
+
+
+@bp.route("/feeds/all.atom")
+@cors_allowed
+@cached(1800)
+def all_atom():
+	feed = _get_all_feed(abs_url_for("feeds.all_atom"))
+	return _atomify(feed)
+
+
+@bp.route("/feeds/packages.json")
+@cors_allowed
+@cached(1800)
+def packages_all_json():
+	feed = _get_new_packages_feed(abs_url_for("feeds.packages_all_json"))
+	return jsonify(feed)
+
+
+@bp.route("/feeds/packages.atom")
+@cors_allowed
+@cached(1800)
+def packages_all_atom():
+	feed = _get_new_packages_feed(abs_url_for("feeds.packages_all_atom"))
+	return _atomify(feed)
+
+
+@bp.route("/feeds/releases.json")
+@cors_allowed
+@cached(1800)
+def releases_all_json():
+	feed = _get_releases_feed(PackageRelease.query, abs_url_for("feeds.releases_all_json"))
+	return jsonify(feed)
+
+
+@bp.route("/feeds/releases.atom")
+@cors_allowed
+@cached(1800)
+def releases_all_atom():
+	feed = _get_releases_feed(PackageRelease.query, abs_url_for("feeds.releases_all_atom"))
+	return _atomify(feed)
+
+
+@bp.route("/packages/<author>/<name>/releases_feed.json")
+@cors_allowed
+@is_package_page
+@cached(1800)
+def releases_package_json(package: Package):
+	feed = _get_releases_feed(package.releases, package.get_url("feeds.releases_package_json", absolute=True))
+	return jsonify(feed)
+
+
+@bp.route("/packages/<author>/<name>/releases_feed.atom")
+@cors_allowed
+@is_package_page
+@cached(1800)
+def releases_package_atom(package: Package):
+	feed = _get_releases_feed(package.releases, package.get_url("feeds.releases_package_atom", absolute=True))
+	return _atomify(feed)

app/blueprints/github/__init__.py

@@ -1,180 +0,0 @@
-# ContentDB
-# Copyright (C) 2018-21 rubenwardy
-#
-# This program is free software: you can redistribute it and/or modify
-# it under the terms of the GNU Affero General Public License as published by
-# the Free Software Foundation, either version 3 of the License, or
-# (at your option) any later version.
-#
-# This program is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-# GNU Affero General Public License for more details.
-#
-# You should have received a copy of the GNU Affero General Public License
-# along with this program.  If not, see <https://www.gnu.org/licenses/>.
-
-from flask import Blueprint, abort
-from flask_babel import gettext
-
-bp = Blueprint("github", __name__)
-
-from flask import redirect, url_for, request, flash, jsonify, current_app
-from flask_login import current_user
-from sqlalchemy import func, or_, and_
-from app import github, csrf
-from app.models import db, User, APIToken, Package, Permission, AuditSeverity, PackageState
-from app.utils import abs_url_for, add_audit_log, login_user_set_active, is_safe_url
-from app.blueprints.api.support import error, api_create_vcs_release
-import hmac, requests
-
-
-@bp.route("/github/start/")
-def start():
-	next = request.args.get("next")
-	if next and not is_safe_url(next):
-		abort(400)
-
-	return github.authorize("", redirect_uri=abs_url_for("github.callback", next=next))
-
-
-@bp.route("/github/view/")
-def view_permissions():
-	url = "https://github.com/settings/connections/applications/" + \
-			current_app.config["GITHUB_CLIENT_ID"]
-	return redirect(url)
-
-
-@bp.route("/github/callback/")
-@github.authorized_handler
-def callback(oauth_token):
-	if oauth_token is None:
-		flash(gettext("Authorization failed [err=gh-oauth-login-failed]"), "danger")
-		return redirect(url_for("users.login"))
-
-	next = request.args.get("next")
-	if next and not is_safe_url(next):
-		abort(400)
-
-	redirect_to = next
-	if redirect_to is None:
-		redirect_to = url_for("homepage.home")
-
-	# Get GitGub username
-	url = "https://api.github.com/user"
-	r = requests.get(url, headers={"Authorization": "token " + oauth_token})
-	username = r.json()["login"]
-
-	# Get user by GitHub username
-	userByGithub = User.query.filter(func.lower(User.github_username) == func.lower(username)).first()
-
-	# If logged in, connect
-	if current_user and current_user.is_authenticated:
-		if userByGithub is None:
-			current_user.github_username = username
-			db.session.commit()
-			flash(gettext("Linked GitHub to account"), "success")
-			return redirect(redirect_to)
-		else:
-			flash(gettext("GitHub account is already associated with another user"), "danger")
-			return redirect(redirect_to)
-
-	# If not logged in, log in
-	else:
-		if userByGithub is None:
-			flash(gettext("Unable to find an account for that GitHub user"), "danger")
-			return redirect(url_for("users.claim_forums"))
-
-		ret = login_user_set_active(userByGithub, next, remember=True)
-		if ret is None:
-			flash(gettext("Authorization failed [err=gh-login-failed]"), "danger")
-			return redirect(url_for("users.login"))
-
-		add_audit_log(AuditSeverity.USER, userByGithub, "Logged in using GitHub OAuth",
-				url_for("users.profile", username=userByGithub.username))
-		db.session.commit()
-		return ret
-
-
-@bp.route("/github/webhook/", methods=["POST"])
-@csrf.exempt
-def webhook():
-	json = request.json
-
-	# Get package
-	github_url = "github.com/" + json["repository"]["full_name"]
-	package = Package.query.filter(
-		Package.repo.ilike("%{}%".format(github_url)), Package.state != PackageState.DELETED).first()
-	if package is None:
-		return error(400, "Could not find package, did you set the VCS repo in CDB correctly? Expected {}".format(github_url))
-
-	# Get all tokens for package
-	tokens_query = APIToken.query.filter(or_(APIToken.package==package,
-			and_(APIToken.package==None, APIToken.owner==package.author)))
-
-	possible_tokens = tokens_query.all()
-	actual_token = None
-
-	#
-	# Check signature
-	#
-
-	header_signature = request.headers.get('X-Hub-Signature')
-	if header_signature is None:
-		return error(403, "Expected payload signature")
-
-	sha_name, signature = header_signature.split('=')
-	if sha_name != 'sha1':
-		return error(403, "Expected SHA1 payload signature")
-
-	for token in possible_tokens:
-		mac = hmac.new(token.access_token.encode("utf-8"), msg=request.data, digestmod='sha1')
-
-		if hmac.compare_digest(str(mac.hexdigest()), signature):
-			actual_token = token
-			break
-
-	if actual_token is None:
-		return error(403, "Invalid authentication, couldn't validate API token")
-
-	if not package.check_perm(actual_token.owner, Permission.APPROVE_RELEASE):
-		return error(403, "You do not have the permission to approve releases")
-
-	#
-	# Check event
-	#
-
-	event = request.headers.get("X-GitHub-Event")
-	if event == "push":
-		ref = json["after"]
-		title = json["head_commit"]["message"].partition("\n")[0]
-		branch = json["ref"].replace("refs/heads/", "")
-		if branch not in [ "master", "main" ]:
-			return jsonify({ "success": False, "message": "Webhook ignored, as it's not on the master/main branch" })
-
-	elif event == "create":
-		ref_type = json.get("ref_type")
-		if ref_type != "tag":
-			return jsonify({
-					"success": False,
-					"message": "Webhook ignored, as it's a non-tag create event. ref_type='{}'.".format(ref_type)
-			})
-
-		ref = json["ref"]
-		title = ref
-
-	elif event == "ping":
-		return jsonify({ "success": True, "message": "Ping successful" })
-
-	else:
-		return error(400, "Unsupported event: '{}'. Only 'push', 'create:tag', and 'ping' are supported."
-				.format(event or "null"))
-
-	#
-	# Perform release
-	#
-
-	if package.releases.filter_by(commit_hash=ref).count() > 0:
-		return
-
-	return api_create_vcs_release(actual_token, package, title, ref, reason="Webhook")

app/blueprints/gitlab/__init__.py

@@ -1,86 +0,0 @@
-# ContentDB
-# Copyright (C) 2020  rubenwardy
-#
-# This program is free software: you can redistribute it and/or modify
-# it under the terms of the GNU Affero General Public License as published by
-# the Free Software Foundation, either version 3 of the License, or
-# (at your option) any later version.
-#
-# This program is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-# GNU Affero General Public License for more details.
-#
-# You should have received a copy of the GNU Affero General Public License
-# along with this program.  If not, see <https://www.gnu.org/licenses/>.
-
-from flask import Blueprint, request, jsonify
-
-bp = Blueprint("gitlab", __name__)
-
-from app import csrf
-from app.models import Package, APIToken, Permission, PackageState
-from app.blueprints.api.support import error, api_create_vcs_release
-
-
-def webhook_impl():
-	json = request.json
-
-	# Get package
-	gitlab_url = json["project"]["web_url"].replace("https://", "").replace("http://", "")
-	package = Package.query.filter(
-		Package.repo.ilike("%{}%".format(gitlab_url)), Package.state != PackageState.DELETED).first()
-	if package is None:
-		return error(400,
-				"Could not find package, did you set the VCS repo in CDB correctly? Expected {}".format(gitlab_url))
-
-	# Get all tokens for package
-	secret = request.headers.get("X-Gitlab-Token")
-	if secret is None:
-		return error(403, "Token required")
-
-	token = APIToken.query.filter_by(access_token=secret).first()
-	if token is None:
-		return error(403, "Invalid authentication")
-
-	if not package.check_perm(token.owner, Permission.APPROVE_RELEASE):
-		return error(403, "You do not have the permission to approve releases")
-
-	#
-	# Check event
-	#
-
-	event = json["event_name"]
-	if event == "push":
-		ref = json["after"]
-		title = ref[:5]
-
-		branch = json["ref"].replace("refs/heads/", "")
-		if branch not in ["master", "main"]:
-			return jsonify({"success": False,
-				"message": "Webhook ignored, as it's not on the master/main branch"})
-
-	elif event == "tag_push":
-		ref = json["ref"]
-		title = ref.replace("refs/tags/", "")
-	else:
-		return error(400, "Unsupported event: '{}'. Only 'push', 'create:tag', and 'ping' are supported."
-				.format(event or "null"))
-
-	#
-	# Perform release
-	#
-
-	if package.releases.filter_by(commit_hash=ref).count() > 0:
-		return
-
-	return api_create_vcs_release(token, package, title, ref, reason="Webhook")
-
-
-@bp.route("/gitlab/webhook/", methods=["POST"])
-@csrf.exempt
-def webhook():
-	try:
-		return webhook_impl()
-	except KeyError as err:
-		return error(400, "Missing field: {}".format(err.args[0]))

app/blueprints/homepage/__init__.py

@@ -18,58 +18,108 @@ from flask import Blueprint, render_template, redirect
 from sqlalchemy import and_
 
 from app.models import Package, PackageReview, Thread, User, PackageState, db, PackageType, PackageRelease, Tags, Tag, \
-	Collection
+	Collection, License, Language
 
 bp = Blueprint("homepage", __name__)
 
-from sqlalchemy.orm import joinedload, subqueryload
+from sqlalchemy.orm import joinedload, subqueryload, load_only, noload
 from sqlalchemy.sql.expression import func
 
 
 PKGS_PER_ROW = 4
 
+# GAMEJAM_BANNER = "https://jam.minetest.net/img/banner.png"
+#
+# class GameJam:
+# 	cover_image = type("", (), dict(url=GAMEJAM_BANNER))()
+# 	tags = []
+#
+# 	def get_cover_image_url(self):
+# 		return GAMEJAM_BANNER
+#
+# 	def get_url(self, _name):
+# 		return "/gamejam/"
+#
+# 	title = "Minetest Game Jam 2023: \"Unexpected\""
+# 	author = None
+#
+# 	short_desc = "The game jam has finished! It's now up to the community to play and rate the games."
+# 	type = type("", (), dict(value="Competition"))()
+# 	content_warnings = []
+# 	reviews = []
+
 
 @bp.route("/gamejam/")
 def gamejam():
-	return redirect("https://forum.minetest.net/viewtopic.php?t=28802")
+	return redirect("https://jam.minetest.net/")
 
 
 @bp.route("/")
 def home():
 	def package_load(query):
 		return query.options(
-				joinedload(Package.author),
+				load_only(Package.name, Package.title, Package.short_desc, Package.state, raiseload=True),
 				subqueryload(Package.main_screenshot),
+				joinedload(Package.author).load_only(User.username, User.display_name, raiseload=True),
+				joinedload(Package.license).load_only(License.name, License.is_foss, raiseload=True),
+				joinedload(Package.media_license).load_only(License.name, License.is_foss, raiseload=True))
+
+	def package_spotlight_load(query):
+		return query.options(
+				load_only(Package.name, Package.title, Package.type, Package.short_desc, Package.state, Package.cover_image_id, raiseload=True),
+				subqueryload(Package.main_screenshot),
+				joinedload(Package.tags),
+				joinedload(Package.content_warnings),
+				joinedload(Package.author).load_only(User.username, User.display_name, raiseload=True),
 				subqueryload(Package.cover_image),
-				joinedload(Package.license),
-				joinedload(Package.media_license))
+				joinedload(Package.license).load_only(License.name, License.is_foss, raiseload=True),
+				joinedload(Package.media_license).load_only(License.name, License.is_foss, raiseload=True))
 
 	def review_load(query):
 		return query.options(
-			joinedload(PackageReview.author),
-			joinedload(PackageReview.thread).subqueryload(Thread.first_reply),
-			joinedload(PackageReview.package).joinedload(Package.author).load_only(User.username, User.display_name),
-			joinedload(PackageReview.package).load_only(Package.title, Package.name).subqueryload(Package.main_screenshot))
+			load_only(PackageReview.id, PackageReview.rating, PackageReview.created_at, PackageReview.language_id, raiseload=True),
+			joinedload(PackageReview.author).load_only(User.username, User.rank, User.email, User.display_name, User.profile_pic, User.is_active, raiseload=True),
+			joinedload(PackageReview.votes),
+			joinedload(PackageReview.language).load_only(Language.title, raiseload=True),
+			joinedload(PackageReview.thread).load_only(Thread.title, Thread.replies_count, raiseload=True).subqueryload(Thread.first_reply),
+			joinedload(PackageReview.package)
+				.load_only(Package.title, Package.name, raiseload=True)
+				.joinedload(Package.author).load_only(User.username, User.display_name, raiseload=True))
 
 	query = Package.query.filter_by(state=PackageState.APPROVED)
-	count = query.count()
+	count = db.session.query(Package.id).filter(Package.state == PackageState.APPROVED).count()
 
-	spotlight_pkgs = query.filter(
-			Package.collections.any(and_(Collection.name == "spotlight", Collection.author.has(username="ContentDB")))) \
-		.order_by(func.random()).limit(6).all()
+	spotlight_pkgs = package_spotlight_load(query.filter(
+			Package.collections.any(and_(Collection.name == "spotlight", Collection.author.has(username="ContentDB"))))
+		.order_by(func.random())).limit(6).all()
+	# spotlight_pkgs.insert(0, GameJam())
 
-	new = package_load(query.order_by(db.desc(Package.approved_at))).limit(PKGS_PER_ROW).all()
-	pop_mod = package_load(query.filter_by(type=PackageType.MOD).order_by(db.desc(Package.score))).limit(2*PKGS_PER_ROW).all()
-	pop_gam = package_load(query.filter_by(type=PackageType.GAME).order_by(db.desc(Package.score))).limit(2*PKGS_PER_ROW).all()
-	pop_txp = package_load(query.filter_by(type=PackageType.TXP).order_by(db.desc(Package.score))).limit(2*PKGS_PER_ROW).all()
-	high_reviewed = package_load(query.order_by(db.desc(Package.score - Package.score_downloads))) \
-			.filter(Package.reviews.any()).limit(PKGS_PER_ROW).all()
+	new = package_load(query).order_by(db.desc(Package.approved_at)).limit(PKGS_PER_ROW).all() # 0.06
+	pop_mod = package_load(query).filter_by(type=PackageType.MOD).order_by(db.desc(Package.score)).limit(2*PKGS_PER_ROW).all()
+	pop_gam = package_load(query).filter_by(type=PackageType.GAME).order_by(db.desc(Package.score)).limit(2*PKGS_PER_ROW).all()
+	pop_txp = package_load(query).filter_by(type=PackageType.TXP).order_by(db.desc(Package.score)).limit(2*PKGS_PER_ROW).all()
 
-	updated = package_load(db.session.query(Package).select_from(PackageRelease).join(Package)
-			.filter_by(state=PackageState.APPROVED)
-			.order_by(db.desc(PackageRelease.releaseDate))
-			.limit(20)).all()
-	updated = updated[:PKGS_PER_ROW]
+	high_reviewed = package_load(query.order_by(db.desc(Package.score - Package.score_downloads))
+			.filter(Package.reviews.any()).limit(PKGS_PER_ROW)).all()
+
+	recent_releases_query = (
+		db.session.query(
+			Package.id,
+			func.max(PackageRelease.created_at).label("max_created_at")
+		)
+		.join(PackageRelease, Package.releases)
+		.group_by(Package.id)
+		.order_by(db.desc("max_created_at"))
+		.limit(3*PKGS_PER_ROW)
+		.subquery())
+
+	updated = (
+		package_load(db.session.query(Package)
+			.select_from(recent_releases_query)
+			.join(Package, Package.id == recent_releases_query.c.id)
+			.filter(Package.state == PackageState.APPROVED)
+			.limit(PKGS_PER_ROW))
+			.all())
 
 	reviews = review_load(PackageReview.query.filter(PackageReview.rating > 3)
 			.order_by(db.desc(PackageReview.created_at))).limit(5).all()

app/blueprints/metrics/__init__.py

@@ -14,64 +14,107 @@
 # You should have received a copy of the GNU Affero General Public License
 # along with this program.  If not, see <https://www.gnu.org/licenses/>.
 
+import datetime
+
 from flask import Blueprint, make_response
+from sqlalchemy import or_, and_
 from sqlalchemy.sql.expression import func
 
-from app.models import Package, db, User, UserRank, PackageState, PackageReview, ThreadReply, Collection
+from app.models import Package, db, User, UserRank, PackageState, PackageReview, ThreadReply, Collection, AuditLogEntry, \
+	PackageTranslation, Language
 from app.rediscache import get_key
 
 bp = Blueprint("metrics", __name__)
 
 
-def generate_metrics(full=False):
+def generate_metrics():
 	def write_single_stat(name, help, type, value):
 		fmt = "# HELP {name} {help}\n# TYPE {name} {type}\n{name} {value}\n\n"
 
 		return fmt.format(name=name, help=help, type=type, value=value)
 
 	def gen_labels(labels):
-		pieces = [key + "=" + str(val) for key, val in labels.items()]
+		pieces = [f"{key}=\"{val}\"" for key, val in labels.items()]
 		return ",".join(pieces)
 
 	def write_array_stat(name, help, type, data):
-		ret = "# HELP {name} {help}\n# TYPE {name} {type}\n" \
+		result = "# HELP {name} {help}\n# TYPE {name} {type}\n" \
 			.format(name=name, help=help, type=type)
 
 		for entry in data:
 			assert(len(entry) == 2)
-			ret += "{name}{{{labels}}} {value}\n" \
+			result += "{name}{{{labels}}} {value}\n" \
 				.format(name=name, labels=gen_labels(entry[0]), value=entry[1])
 
-		return ret + "\n"
+		return result + "\n"
 
 	downloads_result = db.session.query(func.sum(Package.downloads)).one_or_none()
 	downloads = 0 if not downloads_result or not downloads_result[0] else downloads_result[0]
 
 	packages = Package.query.filter_by(state=PackageState.APPROVED).count()
-	users = User.query.filter(User.rank != UserRank.NOT_JOINED).count()
+	users = User.query.filter(User.rank > UserRank.NOT_JOINED, User.rank != UserRank.BOT, User.is_active).count()
+	authors = User.query.filter(User.packages.any(state=PackageState.APPROVED)).count()
+
+	one_day_ago = datetime.datetime.now() - datetime.timedelta(days=1)
+	one_week_ago = datetime.datetime.now() - datetime.timedelta(weeks=1)
+	one_month_ago = datetime.datetime.now() - datetime.timedelta(weeks=4)
+
+	active_users_day = User.query.filter(and_(User.rank != UserRank.BOT, or_(
+		User.audit_log_entries.any(AuditLogEntry.created_at > one_day_ago),
+		User.replies.any(ThreadReply.created_at > one_day_ago)))).count()
+	active_users_week = User.query.filter(and_(User.rank != UserRank.BOT, or_(
+		User.audit_log_entries.any(AuditLogEntry.created_at > one_week_ago),
+		User.replies.any(ThreadReply.created_at > one_week_ago)))).count()
+	active_users_month = User.query.filter(and_(User.rank != UserRank.BOT, or_(
+		User.audit_log_entries.any(AuditLogEntry.created_at > one_month_ago),
+		User.replies.any(ThreadReply.created_at > one_month_ago)))).count()
+
 	reviews = PackageReview.query.count()
 	comments = ThreadReply.query.count()
 	collections = Collection.query.count()
 
+	score_result = db.session.query(func.sum(Package.score)).one_or_none()
+	score = 0 if not score_result or not score_result[0] else score_result[0]
+
+	packages_with_translations = (db.session.query(PackageTranslation.package_id)
+			.filter(PackageTranslation.language_id != "en")
+			.group_by(PackageTranslation.package_id).count())
+	packages_with_translations_meta = (db.session.query(PackageTranslation.package_id)
+			.filter(PackageTranslation.short_desc.is_not(None), PackageTranslation.language_id != "en")
+			.group_by(PackageTranslation.package_id).count())
+	languages_packages = (db.session.query(PackageTranslation.language_id, func.count(Package.id))
+			.select_from(PackageTranslation).outerjoin(Package)
+			.order_by(db.asc(PackageTranslation.language_id))
+			.group_by(PackageTranslation.language_id).all())
+	languages_packages_meta = (db.session.query(PackageTranslation.language_id, func.count(Package.id))
+			.select_from(PackageTranslation).outerjoin(Package)
+			.filter(PackageTranslation.short_desc.is_not(None))
+			.order_by(db.asc(PackageTranslation.language_id))
+			.group_by(PackageTranslation.language_id).all())
+
 	ret = ""
 	ret += write_single_stat("contentdb_packages", "Total packages", "gauge", packages)
 	ret += write_single_stat("contentdb_users", "Number of registered users", "gauge", users)
+	ret += write_single_stat("contentdb_authors", "Number of users with packages", "gauge", authors)
+	ret += write_single_stat("contentdb_users_active_1d", "Number of daily active registered users", "gauge", active_users_day)
+	ret += write_single_stat("contentdb_users_active_1w", "Number of weekly active registered users", "gauge", active_users_week)
+	ret += write_single_stat("contentdb_users_active_1m", "Number of monthly active registered users", "gauge", active_users_month)
 	ret += write_single_stat("contentdb_downloads", "Total downloads", "gauge", downloads)
 	ret += write_single_stat("contentdb_emails", "Number of emails sent", "counter", int(get_key("emails_sent", "0")))
 	ret += write_single_stat("contentdb_reviews", "Number of reviews", "gauge", reviews)
 	ret += write_single_stat("contentdb_comments", "Number of comments", "gauge", comments)
 	ret += write_single_stat("contentdb_collections", "Number of collections", "gauge", collections)
-
-	if full:
-		scores = Package.query.join(User).with_entities(User.username, Package.name, Package.score) \
-			.filter(Package.state==PackageState.APPROVED).all()
-
-		ret += write_array_stat("contentdb_package_score", "Package score", "gauge",
-				[({ "author": score[0], "name": score[1] }, score[2])  for score in scores])
-	else:
-		score_result = db.session.query(func.sum(Package.score)).one_or_none()
-		score = 0 if not score_result or not score_result[0] else score_result[0]
-		ret += write_single_stat("contentdb_score", "Total package score", "gauge", score)
+	ret += write_single_stat("contentdb_score", "Total package score", "gauge", score)
+	ret += write_single_stat("contentdb_packages_with_translations", "Number of packages with translations", "gauge",
+			packages_with_translations)
+	ret += write_single_stat("contentdb_packages_with_translations_meta", "Number of packages with translated meta",
+			"gauge", packages_with_translations_meta)
+	ret += write_array_stat("contentdb_languages_translated",
+			"Number of packages per language", "gauge",
+			[({"language": x[0]}, x[1]) for x in languages_packages])
+	ret += write_array_stat("contentdb_languages_translated_meta",
+			"Number of packages with translated short desc per language", "gauge",
+			[({"language": x[0]}, x[1]) for x in languages_packages_meta])
 
 	return ret
 

app/blueprints/oauth/__init__.py

@@ -15,15 +15,15 @@
 # along with this program.  If not, see <https://www.gnu.org/licenses/>.
 
 import urllib.parse as urlparse
-from typing import Optional
 from urllib.parse import urlencode
 
+import typing
 from flask import Blueprint, render_template, redirect, url_for, request, jsonify, abort, make_response, flash
 from flask_babel import lazy_gettext, gettext
 from flask_login import current_user, login_required
 from flask_wtf import FlaskForm
-from wtforms import StringField, SubmitField, URLField
-from wtforms.validators import InputRequired, Length
+from wtforms import StringField, SubmitField, URLField, SelectField
+from wtforms.validators import InputRequired, Length, Optional
 
 from app import csrf
 from app.blueprints.users.settings import get_setting_tabs
@@ -33,7 +33,7 @@ from app.utils import random_string, add_audit_log
 bp = Blueprint("oauth", __name__)
 
 
-def build_redirect_url(url: str, code: str, state: Optional[str]):
+def build_redirect_url(url: str, code: str, state: typing.Optional[str]):
 	params = {"code": code}
 	if state is not None:
 		params["state"] = state
@@ -51,12 +51,12 @@ def oauth_start():
 	if response_type != "code":
 		return "Unsupported response_type, only code is supported", 400
 
-	client_id = request.args.get("client_id")
-	if client_id is None:
+	client_id = request.args.get("client_id", "")
+	if client_id == "":
 		return "Missing client_id", 400
 
-	redirect_uri = request.args.get("redirect_uri")
-	if redirect_uri is None:
+	redirect_uri = request.args.get("redirect_uri", "")
+	if redirect_uri == "":
 		return "Missing redirect_uri", 400
 
 	client = OAuthClient.query.get_or_404(client_id)
@@ -66,18 +66,14 @@ def oauth_start():
 	if not client.approved and client.owner != current_user:
 		abort(404)
 
-	valid_scopes = {"user:email", "package", "package:release", "package:screenshot"}
-	scope = request.args.get("scope", "")
-	scopes = [x.strip() for x in scope.split(",")]
-	scopes = set([x for x in scopes if x != ""])
-	unknown_scopes = scopes - valid_scopes
-	if unknown_scopes:
-		return f"Unknown scopes: {', '.join(unknown_scopes)}", 400
+	scope = request.args.get("scope", "public")
+	if scope != "public":
+		return "Unsupported scope, only public is supported", 400
 
 	state = request.args.get("state")
 
 	token = APIToken.query.filter(APIToken.client == client, APIToken.owner == current_user).first()
-	if token and not (scopes - token.get_scopes()):
+	if token:
 		token.access_token = random_string(32)
 		token.auth_code = random_string(32)
 		db.session.commit()
@@ -89,19 +85,15 @@ def oauth_start():
 			return redirect(client.redirect_url)
 
 		elif action == "authorize":
-			if token is None:
-				token = APIToken()
-				token.name = f"Token for {client.title} by {client.owner.username}"
-				token.owner = current_user
-				token.client = client
-
+			token = APIToken()
 			token.access_token = random_string(32)
+			token.name = f"Token for {client.title} by {client.owner.username}"
+			token.owner = current_user
+			token.client = client
 			assert client is not None
 			token.auth_code = random_string(32)
 			db.session.add(token)
 
-			token.set_scopes(scopes)
-
 			add_audit_log(AuditSeverity.USER, current_user,
 					f"Granted \"{scope}\" to OAuth2 application \"{client.title}\" by {client.owner.username} [{client_id}] ",
 					url_for("users.profile", username=current_user.username))
@@ -110,42 +102,7 @@ def oauth_start():
 
 			return redirect(build_redirect_url(client.redirect_url, token.auth_code, state))
 
-	scopes_info = []
-	if not scopes:
-		scopes_info.append({
-			"icon": "globe-europe",
-			"title": "Public data only",
-			"description": "Read-only access to your public data",
-		})
-
-	if "user:email" in scopes:
-		scopes_info.append({
-			"icon": "user",
-			"title": gettext("Personal data"),
-			"description": gettext("Email address (read-only)"),
-		})
-
-	if ("package" in scopes or
-			"package:release" in scopes or
-			"package:screenshot" in scopes):
-		if "package" in scopes:
-			msg = gettext("Ability to edit packages and their releases, screenshots, and related data")
-		elif "package:release" in scopes and "package:screenshot" in scopes:
-			msg = gettext("Ability to create and edit releases and screenshots")
-		elif "package:release" in scopes:
-			msg = gettext("Ability to create and edit releases")
-		elif "package:screenshot" in scopes:
-			msg = gettext("Ability to create and edit screenshots")
-		else:
-			assert False, "This should never happen"
-
-		scopes_info.append({
-			"icon": "pen",
-			"title": gettext("Packages"),
-			"description": msg,
-		})
-
-	return render_template("oauth/authorize.html", client=client, scopes=scopes_info)
+	return render_template("oauth/authorize.html", client=client)
 
 
 def error(code: int, msg: str):
@@ -161,16 +118,16 @@ def oauth_grant():
 	if grant_type != "authorization_code":
 		error(400, "Unsupported grant_type, only authorization_code is supported")
 
-	client_id = form.get("client_id")
-	if client_id is None:
+	client_id = form.get("client_id", "")
+	if client_id == "":
 		error(400, "Missing client_id")
 
-	client_secret = form.get("client_secret")
-	if client_secret is None:
+	client_secret = form.get("client_secret", "")
+	if client_secret == "":
 		error(400, "Missing client_secret")
 
-	code = form.get("code")
-	if code is None:
+	code = form.get("code", "")
+	if code == "":
 		error(400, "Missing code")
 
 	client = OAuthClient.query.filter_by(id=client_id, secret=client_secret).first()
@@ -185,6 +142,7 @@ def oauth_grant():
 	db.session.commit()
 
 	return jsonify({
+		"success": True,
 		"access_token": token.access_token,
 		"token_type": "Bearer",
 	})
@@ -208,7 +166,12 @@ def list_clients(username):
 
 class OAuthClientForm(FlaskForm):
 	title = StringField(lazy_gettext("Title"), [InputRequired(), Length(5, 30)])
+	description = StringField(lazy_gettext("Description"), [Optional()])
 	redirect_url = URLField(lazy_gettext("Redirect URL"), [InputRequired(), Length(5, 123)])
+	app_type = SelectField(lazy_gettext("App Type"), [InputRequired()], choices=[
+		("server", "Server-side (client_secret is kept safe)"),
+		("client", "Client-side (client_secret is visible to all users)"),
+	], coerce=lambda x: x)
 	submit = SubmitField(lazy_gettext("Save"))
 
 
@@ -228,6 +191,7 @@ def create_edit_client(username, id_=None):
 			abort(404)
 
 	form = OAuthClientForm(formdata=request.form, obj=client)
+
 	if form.validate_on_submit():
 		if is_new:
 			client = OAuthClient()

app/blueprints/packages/__init__.py

@@ -32,6 +32,11 @@ def get_package_tabs(user: User, package: Package):
 			"title": gettext("Edit Details"),
 			"url": package.get_url("packages.create_edit")
 		},
+		{
+			"id": "translation",
+			"title": gettext("Translation"),
+			"url": package.get_url("packages.translation")
+		},
 		{
 			"id": "releases",
 			"title": gettext("Releases"),
@@ -70,7 +75,7 @@ def get_package_tabs(user: User, package: Package):
 	]
 
 	if package.type == PackageType.MOD or package.type == PackageType.TXP:
-		retval.insert(1, {
+		retval.insert(2, {
 			"id": "game_support",
 			"title": gettext("Supported Games"),
 			"url": package.get_url("packages.game_support")
@@ -79,4 +84,4 @@ def get_package_tabs(user: User, package: Package):
 	return retval
 
 
-from . import packages, screenshots, releases, reviews, game_hub
+from . import packages, advanced_search, screenshots, releases, reviews, game_hub

app/blueprints/packages/advanced_search.py

@@ -0,0 +1,103 @@
+# ContentDB
+# Copyright (C) 2024 rubenwardy
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program.  If not, see <https://www.gnu.org/licenses/>.
+
+from flask import render_template
+from flask_babel import lazy_gettext, gettext
+from flask_wtf import FlaskForm
+from wtforms.fields.choices import SelectField, SelectMultipleField
+from wtforms.fields.simple import StringField, BooleanField
+from wtforms.validators import Optional
+from wtforms_sqlalchemy.fields import QuerySelectMultipleField, QuerySelectField
+
+from . import bp
+from ...models import PackageType, Tag, db, ContentWarning, License, Language, MinetestRelease, Package, PackageState
+
+
+def make_label(obj: Tag | ContentWarning):
+	translated = obj.get_translated()
+	if translated["description"]:
+		return "{}: {}".format(translated["title"], translated["description"])
+	else:
+		return translated["title"]
+
+
+def get_hide_choices():
+	ret = [
+		("android_default", gettext("Android Default")),
+		("desktop_default", gettext("Desktop Default")),
+		("nonfree", gettext("Non-free")),
+		("wip", gettext("Work in Progress")),
+		("deprecated", gettext("Deprecated")),
+		("*", gettext("All content warnings")),
+	]
+	content_warnings = ContentWarning.query.order_by(db.asc(ContentWarning.name)).all()
+	tags = Tag.query.order_by(db.asc(Tag.name)).all()
+	ret += [(x.name, make_label(x)) for x in content_warnings + tags]
+	return ret
+
+
+class AdvancedSearchForm(FlaskForm):
+	q = StringField(lazy_gettext("Query"), [Optional()])
+	type = SelectMultipleField(lazy_gettext("Type"), [Optional()],
+			choices=PackageType.choices(), coerce=PackageType.coerce)
+	author = StringField(lazy_gettext("Author"), [Optional()])
+	tag = QuerySelectMultipleField(lazy_gettext('Tags'),
+			query_factory=lambda: Tag.query.order_by(db.asc(Tag.name)),
+			get_pk=lambda a: a.name, get_label=make_label)
+	flag = QuerySelectMultipleField(lazy_gettext('Content Warnings'),
+			query_factory=lambda: ContentWarning.query.order_by(db.asc(ContentWarning.name)),
+			get_pk=lambda a: a.name, get_label=make_label)
+	license = QuerySelectMultipleField(lazy_gettext("License"), [Optional()],
+			query_factory=lambda: License.query.order_by(db.asc(License.name)),
+			allow_blank=True, blank_value="",
+			get_pk=lambda a: a.name, get_label=lambda a: a.name)
+	game = QuerySelectField(lazy_gettext("Supports Game"), [Optional()],
+			query_factory=lambda: Package.query.filter(Package.type == PackageType.GAME, Package.state == PackageState.APPROVED).order_by(db.asc(Package.name)),
+			allow_blank=True, blank_value="",
+			get_pk=lambda a: f"{a.author.username}/{a.name}",
+			get_label=lambda a: lazy_gettext("%(title)s by %(author)s", title=a.title, author=a.author.display_name))
+	lang = QuerySelectField(lazy_gettext("Language"),
+			query_factory=lambda: Language.query.order_by(db.asc(Language.title)),
+			allow_blank=True, blank_value="",
+			get_pk=lambda a: a.id, get_label=lambda a: a.title)
+	hide = SelectMultipleField(lazy_gettext("Hide Tags and Content Warnings"), [Optional()])
+	engine_version = QuerySelectField(lazy_gettext("Minetest Version"),
+			query_factory=lambda: MinetestRelease.query.order_by(db.asc(MinetestRelease.id)),
+			allow_blank=True, blank_value="",
+			get_pk=lambda a: a.value, get_label=lambda a: a.name)
+	sort = SelectField(lazy_gettext("Sort by"), [Optional()], choices=[
+		("", ""),
+		("name", lazy_gettext("Name")),
+		("title", lazy_gettext("Title")),
+		("score", lazy_gettext("Package score")),
+		("reviews", lazy_gettext("Reviews")),
+		("downloads", lazy_gettext("Downloads")),
+		("created_at", lazy_gettext("Created At")),
+		("approved_at", lazy_gettext("Approved At")),
+		("last_release", lazy_gettext("Last Release")),
+	])
+	order = SelectField(lazy_gettext("Order"), [Optional()], choices=[
+		("desc", lazy_gettext("Descending")),
+		("asc", lazy_gettext("Ascending")),
+	])
+	random = BooleanField(lazy_gettext("Random order"))
+
+
+@bp.route("/packages/advanced-search/")
+def advanced_search():
+	form = AdvancedSearchForm()
+	form.hide.choices = get_hide_choices()
+	return render_template("packages/advanced_search.html", form=form)

app/blueprints/packages/game_hub.py

@@ -44,7 +44,7 @@ def game_hub(package: Package):
 
 	updated = db.session.query(Package).select_from(PackageRelease).join(Package) \
 		.filter(Package.supported_games.any(game=package, supports=True), Package.state==PackageState.APPROVED) \
-		.order_by(db.desc(PackageRelease.releaseDate)) \
+		.order_by(db.desc(PackageRelease.created_at)) \
 		.limit(20).all()
 	updated = updated[:4]
 

app/blueprints/packages/packages.py

@@ -33,10 +33,11 @@ from wtforms_sqlalchemy.fields import QuerySelectField, QuerySelectMultipleField
 from app.logic.LogicError import LogicError
 from app.logic.packages import do_edit_package
 from app.querybuilder import QueryBuilder
-from app.rediscache import has_key, set_key
-from app.tasks.importtasks import import_repo_screenshot, check_zip_release
+from app.rediscache import has_key, set_temp_key
+from app.tasks.importtasks import import_repo_screenshot, check_zip_release, remove_package_game_support, \
+	update_package_game_support
+from app.tasks.pkgtasks import check_package_on_submit
 from app.tasks.webhooktasks import post_discord_webhook
-from app.logic.game_support import GameSupportResolver
 
 from . import bp, get_package_tabs
 from app.models import Package, Tag, db, User, Tags, PackageState, Permission, PackageType, MetaPackage, ForumTopic, \
@@ -44,12 +45,15 @@ from app.models import Package, Tag, db, User, Tags, PackageState, Permission, P
 	PackageScreenshot, NotificationType, AuditLogEntry, PackageAlias, PackageProvides, PackageGameSupport, \
 	PackageDailyStats, Collection
 from app.utils import is_user_bot, get_int_or_abort, is_package_page, abs_url_for, add_audit_log, get_package_by_info, \
-	add_notification, get_system_user, rank_required, get_games_from_csv, get_daterange_options
+	add_notification, get_system_user, rank_required, get_games_from_csv, get_daterange_options, \
+	post_to_approval_thread, normalize_line_endings
+from app.logic.package_approval import validate_package_for_approval, can_move_to_state
+from app.logic.game_support import game_support_set
 
 
 @bp.route("/packages/")
 def list_all():
-	qb    = QueryBuilder(request.args)
+	qb    = QueryBuilder(request.args, cookies=True)
 	query = qb.build_package_query()
 	title = qb.title
 
@@ -65,7 +69,7 @@ def list_all():
 			edited = True
 			key = "tag/{}/{}".format(ip, tag.name)
 			if not has_key(key):
-				set_key(key, "true")
+				set_temp_key(key, "true")
 				Tag.query.filter_by(id=tag.id).update({
 						"views": Tag.views + 1
 					})
@@ -80,7 +84,7 @@ def list_all():
 
 		topic = qb.build_topic_query().first()
 		if qb.search and topic:
-			return redirect("https://forum.minetest.net/viewtopic.php?t=" + str(topic.topic_id))
+			return redirect(topic.url)
 
 	page  = get_int_or_abort(request.args.get("page"), 1)
 	num   = min(40, get_int_or_abort(request.args.get("n"), 100))
@@ -99,7 +103,6 @@ def list_all():
 
 	topics = None
 	if qb.search and not query.has_next:
-		qb.show_discarded = True
 		topics = qb.build_topic_query().all()
 
 	tags_query = db.session.query(func.count(Tags.c.tag_id), Tag) \
@@ -110,7 +113,7 @@ def list_all():
 	selected_tags = set(qb.tags)
 
 	return render_template("packages/list.html",
-			query_hint=title, packages=query.items, pagination=query,
+			query_hint=qb.query_hint, packages=query.items, pagination=query,
 			query=search, tags=tags, selected_tags=selected_tags, type=type_name,
 			authors=authors, packages_count=query.total, topics=topics, noindex=qb.noindex)
 
@@ -133,26 +136,6 @@ def view(package):
 	if not package.check_perm(current_user, Permission.VIEW_PACKAGE):
 		return render_template("packages/gone.html", package=package), 403
 
-	show_similar = not package.approved and (
-			current_user in package.maintainers or
-				package.check_perm(current_user, Permission.APPROVE_NEW))
-
-	conflicting_modnames = None
-	if show_similar and package.type != PackageType.TXP:
-		conflicting_modnames = db.session.query(MetaPackage.name) \
-				.filter(MetaPackage.id.in_([ mp.id for mp in package.provides ])) \
-				.filter(MetaPackage.packages.any(and_(Package.id != package.id, Package.state == PackageState.APPROVED))) \
-				.all()
-
-		conflicting_modnames += db.session.query(ForumTopic.name) \
-				.filter(ForumTopic.name.in_([ mp.name for mp in package.provides ])) \
-				.filter(ForumTopic.topic_id != package.forums) \
-				.filter(~ db.exists().where(Package.forums==ForumTopic.topic_id)) \
-				.order_by(db.asc(ForumTopic.name), db.asc(ForumTopic.title)) \
-				.all()
-
-		conflicting_modnames = set([x[0] for x in conflicting_modnames])
-
 	packages_uses = None
 	if package.type == PackageType.MOD:
 		packages_uses = Package.query.filter(
@@ -169,24 +152,6 @@ def view(package):
 	if review_thread is not None and not review_thread.check_perm(current_user, Permission.SEE_THREAD):
 		review_thread = None
 
-	topic_error = None
-	topic_error_lvl = "warning"
-	if package.state != PackageState.APPROVED and package.forums is not None:
-		errors = []
-		if Package.query.filter(Package.forums==package.forums, Package.state!=PackageState.DELETED).count() > 1:
-			errors.append("<b>" + gettext("Error: Another package already uses this forum topic!") + "</b>")
-			topic_error_lvl = "danger"
-
-		topic = ForumTopic.query.get(package.forums)
-		if topic is not None:
-			if topic.author != package.author:
-				errors.append("<b>" + gettext("Error: Forum topic author doesn't match package author.") + "</b>")
-				topic_error_lvl = "danger"
-		elif package.type != PackageType.TXP:
-			errors.append(gettext("Warning: Forum topic not found. This may happen if the topic has only just been created."))
-
-		topic_error = "<br />".join(errors)
-
 	threads = Thread.query.filter_by(package_id=package.id, review_id=None)
 	if not current_user.is_authenticated:
 		threads = threads.filter_by(private=False)
@@ -196,6 +161,18 @@ def view(package):
 	has_review = current_user.is_authenticated and \
 		PackageReview.query.filter_by(package=package, author=current_user).count() > 0
 
+	validation = None
+	if package.state != PackageState.APPROVED:
+		validation = validate_package_for_approval(package)
+
+	favorites_count = Collection.query.filter(
+			Collection.packages.contains(package),
+			Collection.name == "favorites").count()
+
+	public_collection_count = Collection.query.filter(
+		Collection.packages.contains(package),
+		Collection.private == False).count()
+
 	is_favorited = current_user.is_authenticated and \
 		Collection.query.filter(
 			Collection.author == current_user,
@@ -204,9 +181,9 @@ def view(package):
 
 	return render_template("packages/view.html",
 			package=package, releases=releases, packages_uses=packages_uses,
-			conflicting_modnames=conflicting_modnames,
-			review_thread=review_thread, topic_error=topic_error, topic_error_lvl=topic_error_lvl,
-			threads=threads.all(), has_review=has_review, is_favorited=is_favorited)
+			review_thread=review_thread, threads=threads.all(), validation=validation,
+			has_review=has_review, favorites_count=favorites_count, is_favorited=is_favorited,
+			public_collection_count=public_collection_count)
 
 
 @bp.route("/packages/<author>/<name>/shields/<type>/")
@@ -242,11 +219,12 @@ def download(package):
 		return redirect(release.get_download_url())
 
 
-def makeLabel(obj):
-	if obj.description:
-		return "{}: {}".format(obj.title, obj.description)
+def make_label(obj: Tag | ContentWarning):
+	translated = obj.get_translated()
+	if translated["description"]:
+		return "{}: {}".format(translated["title"], translated["description"])
 	else:
-		return obj.title
+		return translated["title"]
 
 
 class PackageForm(FlaskForm):
@@ -257,12 +235,12 @@ class PackageForm(FlaskForm):
 
 	dev_state        = SelectField(lazy_gettext("Maintenance State"), [InputRequired()], choices=PackageDevState.choices(with_none=True), coerce=PackageDevState.coerce)
 
-	tags             = QuerySelectMultipleField(lazy_gettext('Tags'), query_factory=lambda: Tag.query.order_by(db.asc(Tag.name)), get_pk=lambda a: a.id, get_label=makeLabel)
-	content_warnings = QuerySelectMultipleField(lazy_gettext('Content Warnings'), query_factory=lambda: ContentWarning.query.order_by(db.asc(ContentWarning.name)), get_pk=lambda a: a.id, get_label=makeLabel)
+	tags             = QuerySelectMultipleField(lazy_gettext('Tags'), query_factory=lambda: Tag.query.order_by(db.asc(Tag.name)), get_pk=lambda a: a.id, get_label=make_label)
+	content_warnings = QuerySelectMultipleField(lazy_gettext('Content Warnings'), query_factory=lambda: ContentWarning.query.order_by(db.asc(ContentWarning.name)), get_pk=lambda a: a.id, get_label=make_label)
 	license          = QuerySelectField(lazy_gettext("License"), [DataRequired()], allow_blank=True, query_factory=lambda: License.query.order_by(db.asc(License.name)), get_pk=lambda a: a.id, get_label=lambda a: a.name)
 	media_license    = QuerySelectField(lazy_gettext("Media License"), [DataRequired()], allow_blank=True, query_factory=lambda: License.query.order_by(db.asc(License.name)), get_pk=lambda a: a.id, get_label=lambda a: a.name)
 
-	desc             = TextAreaField(lazy_gettext("Long Description (Markdown)"), [Optional(), Length(0,10000)])
+	desc             = TextAreaField(lazy_gettext("Long Description (Markdown)"), [Optional(), Length(0,10000)], filters=[normalize_line_endings])
 
 	repo             = StringField(lazy_gettext("VCS Repository URL"), [Optional(), URL()], filters = [lambda x: x or None])
 	website          = StringField(lazy_gettext("Website URL"), [Optional(), URL()], filters = [lambda x: x or None])
@@ -270,6 +248,7 @@ class PackageForm(FlaskForm):
 	forums           = IntegerField(lazy_gettext("Forum Topic ID"), [Optional(), NumberRange(0, 999999)])
 	video_url        = StringField(lazy_gettext("Video URL"), [Optional(), URL()], filters=[lambda x: x or None])
 	donate_url       = StringField(lazy_gettext("Donate URL"), [Optional(), URL()], filters=[lambda x: x or None])
+	translation_url  = StringField(lazy_gettext("Translation URL"), [Optional(), URL()], filters=[lambda x: x or None])
 
 	submit           = SubmitField(lazy_gettext("Save"))
 
@@ -323,6 +302,7 @@ def handle_create_edit(package: typing.Optional[Package], form: PackageForm, aut
 			"forums": form.forums.data,
 			"video_url": form.video_url.data,
 			"donate_url": form.donate_url.data,
+			"translation_url": form.translation_url.data,
 		})
 
 		if wasNew:
@@ -347,9 +327,15 @@ def handle_create_edit(package: typing.Optional[Package], form: PackageForm, aut
 @bp.route("/packages/<author>/<name>/edit/", methods=["GET", "POST"])
 @login_required
 def create_edit(author=None, name=None):
+	if current_user.email is None:
+		flash(gettext("You must add an email address to your account and confirm it before you can manage packages"), "danger")
+		return redirect(url_for("users.email_notifications"))
+
 	package = None
 	if author is None:
 		form = PackageForm(formdata=request.form)
+		form.submit.label.text = lazy_gettext("Save draft")
+
 		author = request.args.get("author")
 		if author is None or author == current_user.username:
 			author = current_user
@@ -368,7 +354,7 @@ def create_edit(author=None, name=None):
 		if package is None:
 			abort(404)
 		if not package.check_perm(current_user, Permission.EDIT_PACKAGE):
-			return redirect(package.get_url("packages.view"))
+			abort(403)
 
 		author = package.author
 
@@ -415,10 +401,14 @@ def move_to_state(package):
 	if state is None:
 		abort(400)
 
-	if not package.can_move_to_state(current_user, state):
+	if package.state == state:
+		return redirect(package.get_url("packages.view"))
+
+	if not can_move_to_state(package, current_user, state):
 		flash(gettext("You don't have permission to do that"), "danger")
 		return redirect(package.get_url("packages.view"))
 
+	old_state = package.state
 	package.state = state
 	msg = "Marked {} as {}".format(package.title, state.value)
 
@@ -426,7 +416,7 @@ def move_to_state(package):
 		if not package.approved_at:
 			post_discord_webhook.delay(package.author.display_name,
 					"New package {}".format(package.get_url("packages.view", absolute=True)), False,
-					package.title, package.short_desc, package.get_thumb_url(2, True))
+					package.title, package.short_desc, package.get_thumb_url(2, True, "png"))
 			package.approved_at = datetime.datetime.now()
 
 		screenshots = PackageScreenshot.query.filter_by(package=package, approved=False).all()
@@ -434,36 +424,77 @@ def move_to_state(package):
 			s.approved = True
 
 		msg = "Approved {}".format(package.title)
+		update_package_game_support.delay(package.id)
 	elif state == PackageState.READY_FOR_REVIEW:
 		post_discord_webhook.delay(package.author.display_name,
 				"Ready for Review: {}".format(package.get_url("packages.view", absolute=True)), True,
-				package.title, package.short_desc, package.get_thumb_url(2, True))
+				package.title, package.short_desc, package.get_thumb_url(2, True, "png"))
 
 	add_notification(package.maintainers, current_user, NotificationType.PACKAGE_APPROVAL, msg, package.get_url("packages.view"), package)
 	severity = AuditSeverity.NORMAL if current_user in package.maintainers else AuditSeverity.EDITOR
 	add_audit_log(severity, current_user, msg, package.get_url("packages.view"), package)
+	post_to_approval_thread(package, current_user, msg, True)
 
 	db.session.commit()
 
+	check_package_on_submit.delay(package.id)
+
 	if package.state == PackageState.CHANGES_NEEDED:
 		flash(gettext("Please comment what changes are needed in the approval thread"), "warning")
 		if package.review_thread:
 			return redirect(package.review_thread.get_view_url())
 		else:
 			return redirect(url_for('threads.new', pid=package.id, title='Package approval comments'))
+	elif (package.review_thread and
+			old_state == PackageState.CHANGES_NEEDED and package.state == PackageState.READY_FOR_REVIEW):
+		flash(gettext("Please comment in the approval thread so editors know what you have changed"), "warning")
+		return redirect(package.review_thread.get_view_url())
 
 	return redirect(package.get_url("packages.view"))
 
 
+@bp.route("/packages/<author>/<name>/translation/")
+@login_required
+@is_package_page
+def translation(package):
+	return render_template("packages/translation.html", package=package,
+			tabs=get_package_tabs(current_user, package), current_tab="translation")
+
+
 @bp.route("/packages/<author>/<name>/remove/", methods=["GET", "POST"])
 @login_required
 @is_package_page
 def remove(package):
+	if not package.check_perm(current_user, Permission.EDIT_PACKAGE):
+		abort(403)
+
+	states = [PackageDevState.AS_IS, PackageDevState.DEPRECATED, PackageDevState.LOOKING_FOR_MAINTAINER]
+
 	if request.method == "GET":
-		return render_template("packages/remove.html", package=package,
+		# Find packages that will having missing hard deps after this action
+		broken_meta = MetaPackage.query.filter(MetaPackage.packages.contains(package),
+				~MetaPackage.packages.any(and_(Package.id != package.id, Package.state == PackageState.APPROVED)))
+		hard_deps = Package.query.filter(
+			Package.state == PackageState.APPROVED,
+			Package.dependencies.any(
+				and_(Dependency.meta_package_id.in_([x.id for x in broken_meta]), Dependency.optional == False))).all()
+
+		return render_template("packages/remove.html",
+				package=package, hard_deps=hard_deps, states=states,
 				tabs=get_package_tabs(current_user, package), current_tab="remove")
 
+	for state in states:
+		if state.name in request.form:
+			flash(gettext("Set state to %(state)s", state=state.title), "success")
+			package.dev_state = state
+			msg = "Set dev state of {} to {}".format(package.title, state.title)
+			add_audit_log(AuditSeverity.NORMAL, current_user, msg, package.get_url("packages.view"), package)
+			db.session.commit()
+			return redirect(package.get_url("packages.view"))
+
 	reason = request.form.get("reason") or "?"
+	if len(reason) > 500:
+		abort(400)
 
 	if "delete" in request.form:
 		if not package.check_perm(current_user, Permission.DELETE_PACKAGE):
@@ -480,7 +511,9 @@ def remove(package):
 
 		post_discord_webhook.delay(current_user.username,
 			f"Deleted package {package.author.username}/{package.name} with reason '{reason}'",
-			True, package.title, package.short_desc, package.get_thumb_url(2, True))
+			True, package.title, package.short_desc, package.get_thumb_url(2, True, "png"))
+
+		remove_package_game_support.delay(package.id)
 
 		flash(gettext("Deleted package"), "success")
 
@@ -500,7 +533,9 @@ def remove(package):
 
 		post_discord_webhook.delay(current_user.username,
 			"Unapproved package with reason {}\n\n{}".format(reason, package.get_url("packages.view", absolute=True)), True,
-			package.title, package.short_desc, package.get_thumb_url(2, True))
+			package.title, package.short_desc, package.get_thumb_url(2, True, "png"))
+
+		remove_package_game_support.delay(package.id)
 
 		flash(gettext("Unapproved package"), "success")
 
@@ -674,10 +709,27 @@ def similar(package):
 			packages_modnames=packages_modnames, similar_topics=similar_topics)
 
 
+def csv_games_check(_form, field):
+	game_names = [name.strip() for name in field.data.split(",")]
+	if len(game_names) == 0 or (len(game_names) == 1 and game_names[0] == ""):
+		return
+
+	missing = set()
+	for game_name in game_names:
+		if game_name.endswith("_game"):
+			game_name = game_name[:-5]
+		if Package.query.filter(and_(Package.state==PackageState.APPROVED, Package.type==PackageType.GAME,
+				or_(Package.name==game_name, Package.name==game_name + "_game"))).count() == 0:
+			missing.add(game_name)
+
+	if len(missing) > 0:
+		raise ValidationError(f"Unable to find game {','.join(missing)}")
+
+
 class GameSupportForm(FlaskForm):
 	enable_support_detection = BooleanField(lazy_gettext("Enable support detection based on dependencies (recommended)"), [Optional()])
-	supported = StringField(lazy_gettext("Supported games"), [Optional()])
-	unsupported = StringField(lazy_gettext("Unsupported games"), [Optional()])
+	supported = StringField(lazy_gettext("Supported games"), [Optional(), csv_games_check])
+	unsupported = StringField(lazy_gettext("Unsupported games"), [Optional(), csv_games_check])
 	supports_all_games = BooleanField(lazy_gettext("Supports all games (unless stated) / is game independent"), [Optional()])
 	submit = SubmitField(lazy_gettext("Save"))
 
@@ -728,14 +780,12 @@ def game_support(package):
 
 		if can_override:
 			try:
-				resolver = GameSupportResolver(db.session)
-
 				game_is_supported = {}
 				for game in get_games_from_csv(db.session, form.supported.data or ""):
 					game_is_supported[game.id] = True
 				for game in get_games_from_csv(db.session, form.unsupported.data or ""):
 					game_is_supported[game.id] = False
-				resolver.set_supported(package, game_is_supported, 11)
+				game_support_set(db.session, package, game_is_supported, 11)
 				detect_update_needed = True
 			except LogicError as e:
 				flash(e.message, "danger")

app/blueprints/packages/releases.py

@@ -13,21 +13,23 @@
 #
 # You should have received a copy of the GNU Affero General Public License
 # along with this program.  If not, see <https://www.gnu.org/licenses/>.
+import os
 
 from flask import render_template, request, redirect, flash, url_for, abort
 from flask_babel import lazy_gettext, gettext
 from flask_login import login_required, current_user
 from flask_wtf import FlaskForm
 from wtforms import StringField, SubmitField, BooleanField, RadioField, FileField
+from wtforms.fields.simple import TextAreaField
 from wtforms.validators import InputRequired, Length, Optional
 from wtforms_sqlalchemy.fields import QuerySelectField
 
 from app.logic.releases import do_create_vcs_release, LogicError, do_create_zip_release
 from app.models import Package, db, User, PackageState, Permission, UserRank, PackageDailyStats, MinetestRelease, \
 	PackageRelease, PackageUpdateTrigger, PackageUpdateConfig
-from app.rediscache import has_key, set_key, make_download_key
+from app.rediscache import has_key, set_temp_key, make_download_key
 from app.tasks.importtasks import check_update_config
-from app.utils import is_user_bot, is_package_page, nonempty_or_none
+from app.utils import is_user_bot, is_package_page, nonempty_or_none, normalize_line_endings
 from . import bp, get_package_tabs
 
 
@@ -50,19 +52,25 @@ def get_mt_releases(is_max):
 
 
 class CreatePackageReleaseForm(FlaskForm):
-	title       = StringField(lazy_gettext("Title"), [InputRequired(), Length(1, 30)])
-	uploadOpt   = RadioField(lazy_gettext("Method"), choices=[("upload", lazy_gettext("File Upload"))], default="upload")
-	vcsLabel    = StringField(lazy_gettext("Git reference (ie: commit hash, branch, or tag)"), default=None)
+	name        = StringField(lazy_gettext("Name"), [InputRequired(), Length(1, 30)])
+	title       = StringField(lazy_gettext("Title"), [Optional(), Length(1, 100)], filters=[nonempty_or_none])
+	release_notes = TextAreaField(lazy_gettext("Release Notes"), [Optional(), Length(1, 5000)],
+			filters=[nonempty_or_none, normalize_line_endings])
+	upload_mode = RadioField(lazy_gettext("Method"), choices=[("upload", lazy_gettext("File Upload"))], default="upload")
+	vcs_label   = StringField(lazy_gettext("Git reference (ie: commit hash, branch, or tag)"), default=None)
 	file_upload = FileField(lazy_gettext("File Upload"))
 	min_rel     = QuerySelectField(lazy_gettext("Minimum Minetest Version"), [InputRequired()],
 			query_factory=lambda: get_mt_releases(False), get_pk=lambda a: a.id, get_label=lambda a: a.name)
-	max_rel    = QuerySelectField(lazy_gettext("Maximum Minetest Version"), [InputRequired()],
+	max_rel     = QuerySelectField(lazy_gettext("Maximum Minetest Version"), [InputRequired()],
 			query_factory=lambda: get_mt_releases(True), get_pk=lambda a: a.id, get_label=lambda a: a.name)
-	submit     = SubmitField(lazy_gettext("Save"))
+	submit      = SubmitField(lazy_gettext("Save"))
 
 
 class EditPackageReleaseForm(FlaskForm):
-	title    = StringField(lazy_gettext("Title"), [InputRequired(), Length(1, 30)])
+	name     = StringField(lazy_gettext("Name"), [InputRequired(), Length(1, 30)])
+	title    = StringField(lazy_gettext("Title"), [InputRequired(), Length(1, 30)], filters=[nonempty_or_none])
+	release_notes = TextAreaField(lazy_gettext("Release Notes"), [Optional(), Length(1, 5000)],
+			filters=[nonempty_or_none, normalize_line_endings])
 	url      = StringField(lazy_gettext("URL"), [Optional()])
 	task_id  = StringField(lazy_gettext("Task ID"), filters = [lambda x: x or None])
 	approved = BooleanField(lazy_gettext("Is Approved"))
@@ -77,27 +85,31 @@ class EditPackageReleaseForm(FlaskForm):
 @login_required
 @is_package_page
 def create_release(package):
+	if current_user.email is None:
+		flash(gettext("You must add an email address to your account and confirm it before you can manage packages"), "danger")
+		return redirect(url_for("users.email_notifications"))
+
 	if not package.check_perm(current_user, Permission.MAKE_RELEASE):
 		return redirect(package.get_url("packages.view"))
 
 	# Initial form class from post data and default data
 	form = CreatePackageReleaseForm()
 	if package.repo is not None:
-		form["uploadOpt"].choices = [("vcs", gettext("Import from Git")), ("upload", gettext("Upload .zip file"))]
+		form.upload_mode.choices = [("vcs", gettext("Import from Git")), ("upload", gettext("Upload .zip file"))]
 		if request.method == "GET":
-			form["uploadOpt"].data = "vcs"
-			form.vcsLabel.data = request.args.get("ref")
+			form.upload_mode.data = "vcs"
+			form.vcs_label.data = request.args.get("ref")
 
 	if request.method == "GET":
 		form.title.data = request.args.get("title")
 
 	if form.validate_on_submit():
 		try:
-			if form["uploadOpt"].data == "vcs":
-				rel = do_create_vcs_release(current_user, package, form.title.data,
-						form.vcsLabel.data, form.min_rel.data.get_actual(), form.max_rel.data.get_actual())
+			if form.upload_mode.data == "vcs":
+				rel = do_create_vcs_release(current_user, package, form.name.data, form.title.data, form.release_notes.data,
+						form.vcs_label.data, form.min_rel.data.get_actual(), form.max_rel.data.get_actual())
 			else:
-				rel = do_create_zip_release(current_user, package, form.title.data,
+				rel = do_create_zip_release(current_user, package, form.name.data, form.title.data, form.release_notes.data,
 						form.file_upload.data, form.min_rel.data.get_actual(), form.max_rel.data.get_actual())
 			return redirect(url_for("tasks.check", id=rel.task_id, r=rel.get_edit_url()))
 		except LogicError as e:
@@ -121,7 +133,7 @@ def download_release(package, id):
 
 		key = make_download_key(ip, release.package)
 		if not has_key(key):
-			set_key(key, "true")
+			set_temp_key(key, "true")
 
 			bonus = 0
 			if reason == "new":
@@ -144,11 +156,21 @@ def download_release(package, id):
 	return redirect(release.url)
 
 
-@bp.route("/packages/<author>/<name>/releases/<int:id>/", methods=["GET", "POST"])
+@bp.route("/packages/<author>/<name>/releases/<int:id>/")
+@is_package_page
+def view_release(package, id):
+	release: PackageRelease = PackageRelease.query.get(id)
+	if release is None or release.package != package:
+		abort(404)
+
+	return render_template("packages/release_view.html", package=package, release=release)
+
+
+@bp.route("/packages/<author>/<name>/releases/<int:id>/edit/", methods=["GET", "POST"])
 @login_required
 @is_package_page
 def edit_release(package, id):
-	release : PackageRelease = PackageRelease.query.get(id)
+	release: PackageRelease = PackageRelease.query.get(id)
 	if release is None or release.package != package:
 		abort(404)
 
@@ -166,13 +188,15 @@ def edit_release(package, id):
 
 	if form.validate_on_submit():
 		if canEdit:
-			release.title = form["title"].data
-			release.min_rel = form["min_rel"].data.get_actual()
-			release.max_rel = form["max_rel"].data.get_actual()
+			release.name = form.name.data
+			release.title = form.title.data
+			release.release_notes = form.release_notes.data
+			release.min_rel = form.min_rel.data.get_actual()
+			release.max_rel = form.max_rel.data.get_actual()
 
 		if package.check_perm(current_user, Permission.CHANGE_RELEASE_URL):
-			release.url = form["url"].data
-			release.task_id = form["task_id"].data
+			release.url = form.url.data
+			release.task_id = form.task_id.data
 			if release.task_id is not None:
 				release.task_id = None
 
@@ -215,10 +239,10 @@ def bulk_change_release(package):
 		only_change_none = form.only_change_none.data
 
 		for release in package.releases.all():
-			if form["set_min"].data and (not only_change_none or release.min_rel is None):
-				release.min_rel = form["min_rel"].data.get_actual()
-			if form["set_max"].data and (not only_change_none or release.max_rel is None):
-				release.max_rel = form["max_rel"].data.get_actual()
+			if form.set_min.data and (not only_change_none or release.min_rel is None):
+				release.min_rel = form.min_rel.data.get_actual()
+			if form.set_max.data and (not only_change_none or release.max_rel is None):
+				release.max_rel = form.max_rel.data.get_actual()
 
 		db.session.commit()
 
@@ -241,6 +265,9 @@ def delete_release(package, id):
 	db.session.delete(release)
 	db.session.commit()
 
+	if release.file_path and os.path.isfile(release.file_path):
+		os.remove(release.file_path)
+
 	return redirect(package.get_url("packages.view"))
 
 

app/blueprints/packages/reviews.py

@@ -18,17 +18,18 @@ from collections import namedtuple
 
 import typing
 from flask import render_template, request, redirect, flash, url_for, abort, jsonify
-from flask_babel import gettext, lazy_gettext
+from flask_babel import gettext, lazy_gettext, get_locale
 from flask_login import current_user, login_required
 from flask_wtf import FlaskForm
 from wtforms import StringField, TextAreaField, SubmitField, RadioField
-from wtforms.validators import InputRequired, Length
+from wtforms.validators import InputRequired, Length, DataRequired
+from wtforms_sqlalchemy.fields import QuerySelectField
 
 from app.models import db, PackageReview, Thread, ThreadReply, NotificationType, PackageReviewVote, Package, UserRank, \
-	Permission, AuditSeverity, PackageState
+	Permission, AuditSeverity, PackageState, Language
 from app.tasks.webhooktasks import post_discord_webhook
 from app.utils import is_package_page, add_notification, get_int_or_abort, is_yes, is_safe_url, rank_required, \
-	add_audit_log, has_blocked_domains, should_return_json
+	add_audit_log, has_blocked_domains, should_return_json, normalize_line_endings
 from . import bp
 
 
@@ -41,9 +42,22 @@ def list_reviews():
 	return render_template("packages/reviews_list.html", pagination=pagination, reviews=pagination.items)
 
 
+def get_default_language():
+	locale = get_locale()
+	if locale:
+		return Language.query.filter_by(id=locale.language).first()
+
+	return None
+
 class ReviewForm(FlaskForm):
 	title = StringField(lazy_gettext("Title"), [InputRequired(), Length(3, 100)])
-	comment = TextAreaField(lazy_gettext("Comment"), [InputRequired(), Length(10, 2000)])
+	language = QuerySelectField(lazy_gettext("Language"), [DataRequired()],
+			allow_blank=True,
+			query_factory=lambda: Language.query.order_by(db.asc(Language.title)),
+			get_pk=lambda a: a.id,
+			get_label=lambda a: a.title,
+			default=get_default_language)
+	comment = TextAreaField(lazy_gettext("Comment"), [InputRequired(), Length(10, 2000)], filters=[normalize_line_endings])
 	rating = RadioField(lazy_gettext("Rating"), [InputRequired()],
 			choices=[("5", lazy_gettext("Yes")), ("3", lazy_gettext("Neutral")), ("1", lazy_gettext("No"))])
 	btn_submit = SubmitField(lazy_gettext("Save"))
@@ -88,6 +102,7 @@ def review(package):
 				db.session.add(review)
 
 			review.rating = int(form.rating.data)
+			review.language = form.language.data
 
 			thread = review.thread
 			if not thread:
@@ -128,8 +143,8 @@ def review(package):
 							 url_for("threads.view", id=thread.id), package)
 
 			if was_new:
-				post_discord_webhook.delay(thread.author.display_name,
-						"Reviewed {}: {}".format(package.title, thread.get_view_url(absolute=True)), False)
+				msg = f"Reviewed {package.title} ({review.language.title}): {thread.get_view_url(absolute=True)}"
+				post_discord_webhook.delay(thread.author.display_name, msg, False)
 
 			db.session.commit()
 
@@ -245,15 +260,17 @@ def review_votes(package):
 			else:
 				user_biases[vote.user.username][1] += 1
 
+	reviews = package.reviews.all()
+
 	BiasInfo = namedtuple("BiasInfo", "username balance with_ against no_vote perc_with")
 	user_biases_info = []
 	for username, bias in user_biases.items():
 		total_votes = bias[0] + bias[1]
 		balance = bias[0] - bias[1]
 		perc_with = round((100 * bias[0]) / total_votes)
-		user_biases_info.append(BiasInfo(username, balance, bias[0], bias[1], len(package.reviews) - total_votes, perc_with))
+		user_biases_info.append(BiasInfo(username, balance, bias[0], bias[1], len(reviews) - total_votes, perc_with))
 
 	user_biases_info.sort(key=lambda x: -abs(x.balance))
 
-	return render_template("packages/review_votes.html", package=package, reviews=package.reviews,
+	return render_template("packages/review_votes.html", package=package, reviews=reviews,
 			user_biases=user_biases_info)

app/blueprints/packages/screenshots.py

@@ -13,6 +13,7 @@
 #
 # You should have received a copy of the GNU Affero General Public License
 # along with this program.  If not, see <https://www.gnu.org/licenses/>.
+import os
 
 from flask import render_template, request, redirect, flash, url_for, abort
 from flask_babel import lazy_gettext, gettext
@@ -111,10 +112,10 @@ def edit_screenshot(package, id):
 		was_approved = screenshot.approved
 
 		if can_edit:
-			screenshot.title = form["title"].data or "Untitled"
+			screenshot.title = form.title.data or "Untitled"
 
 		if can_approve:
-			screenshot.approved = form["approved"].data
+			screenshot.approved = form.approved.data
 		else:
 			screenshot.approved = was_approved
 
@@ -143,4 +144,6 @@ def delete_screenshot(package, id):
 	db.session.delete(screenshot)
 	db.session.commit()
 
+	os.remove(screenshot.file_path)
+
 	return redirect(package.get_url("packages.screenshots"))

app/blueprints/report/__init__.py

@@ -25,13 +25,13 @@ from wtforms.validators import InputRequired, Length
 from app.models import User, UserRank
 from app.tasks.emails import send_user_email
 from app.tasks.webhooktasks import post_discord_webhook
-from app.utils import is_no, abs_url_samesite
+from app.utils import is_no, abs_url_samesite, normalize_line_endings
 
 bp = Blueprint("report", __name__)
 
 
 class ReportForm(FlaskForm):
-	message = TextAreaField(lazy_gettext("Message"), [InputRequired(), Length(10, 10000)])
+	message = TextAreaField(lazy_gettext("Message"), [InputRequired(), Length(10, 10000)], filters=[normalize_line_endings])
 	submit = SubmitField(lazy_gettext("Report"))
 
 

app/blueprints/tasks/__init__.py

@@ -55,7 +55,10 @@ def check(id):
 		if current_user.is_authenticated and current_user.rank.at_least(UserRank.ADMIN):
 			info["error"] = str(traceback)
 		elif str(result)[1:12] == "TaskError: ":
-			info["error"] = str(result)[12:-1]
+			if hasattr(result, "value"):
+				info["error"] = result.value
+			else:
+				info["error"] = str(result)
 		else:
 			info["error"] = "Unknown server error"
 	else:

app/blueprints/threads/__init__.py

@@ -16,8 +16,7 @@
 
 from flask import Blueprint, request, render_template, abort, flash, redirect, url_for
 from flask_babel import gettext, lazy_gettext
-from sqlalchemy import or_
-from sqlalchemy.orm import selectinload, joinedload
+from sqlalchemy.orm import selectinload
 
 from app.markdown import get_user_mentions, render_markdown
 from app.tasks.webhooktasks import post_discord_webhook
@@ -27,7 +26,8 @@ bp = Blueprint("threads", __name__)
 from flask_login import current_user, login_required
 from app.models import Package, db, User, Permission, Thread, UserRank, AuditSeverity, \
 	NotificationType, ThreadReply
-from app.utils import add_notification, is_yes, add_audit_log, get_system_user, has_blocked_domains
+from app.utils import add_notification, is_yes, add_audit_log, get_system_user, has_blocked_domains, \
+	normalize_line_endings
 from flask_wtf import FlaskForm
 from wtforms import StringField, TextAreaField, SubmitField, BooleanField
 from wtforms.validators import InputRequired, Length
@@ -178,7 +178,7 @@ def delete_reply(id):
 
 
 class CommentForm(FlaskForm):
-	comment = TextAreaField(lazy_gettext("Comment"), [InputRequired(), Length(2, 2000)])
+	comment = TextAreaField(lazy_gettext("Comment"), [InputRequired(), Length(2, 2000)], filters=[normalize_line_endings])
 	btn_submit = SubmitField(lazy_gettext("Comment"))
 
 
@@ -258,7 +258,8 @@ def view(id):
 			add_notification(mentioned, current_user, NotificationType.THREAD_REPLY,
 							 msg, thread.get_view_url(), thread.package)
 
-			thread.watchers.append(mentioned)
+			if mentioned not in thread.watchers:
+				thread.watchers.append(mentioned)
 
 		msg = "New comment on '{}'".format(thread.title)
 		add_notification(thread.watchers, current_user, NotificationType.THREAD_REPLY, msg, thread.get_view_url(), thread.package)
@@ -279,7 +280,7 @@ def view(id):
 
 class ThreadForm(FlaskForm):
 	title	= StringField(lazy_gettext("Title"), [InputRequired(), Length(3,100)])
-	comment = TextAreaField(lazy_gettext("Comment"), [InputRequired(), Length(10, 2000)])
+	comment = TextAreaField(lazy_gettext("Comment"), [InputRequired(), Length(10, 2000)], filters=[normalize_line_endings])
 	private = BooleanField(lazy_gettext("Private"))
 	btn_submit  = SubmitField(lazy_gettext("Open Thread"))
 

app/blueprints/thumbnails/__init__.py

@@ -14,15 +14,19 @@
 # You should have received a copy of the GNU Affero General Public License
 # along with this program.  If not, see <https://www.gnu.org/licenses/>.
 
-
-from flask import abort, send_file, Blueprint, current_app
-
-bp = Blueprint("thumbnails", __name__)
-
+import re
+import requests
+from flask import abort, send_file, Blueprint, current_app, request
 import os
 from PIL import Image
 
-ALLOWED_RESOLUTIONS=[(100,67), (270,180), (350,233), (1100,520)]
+
+bp = Blueprint("thumbnails", __name__)
+
+
+ALLOWED_RESOLUTIONS = [(100, 67), (270, 180), (350, 233), (1100, 520)]
+ALLOWED_EXTENSIONS = {"png", "webp", "jpg"}
+
 
 def mkdir(path):
 	assert path != "" and path is not None
@@ -34,34 +38,53 @@ def mkdir(path):
 
 
 def resize_and_crop(img_path, modified_path, size):
-	try:
-		img = Image.open(img_path)
-	except FileNotFoundError:
+	with Image.open(img_path) as img:
+		# Get current and desired ratio for the images
+		img_ratio = img.size[0] / float(img.size[1])
+		desired_ratio = size[0] / float(size[1])
+
+		# Is more portrait than target, scale and crop
+		if desired_ratio > img_ratio:
+			img = img.resize((int(size[0]), int(size[0] * img.size[1] / img.size[0])),
+					Image.BICUBIC)
+			box = (0, (img.size[1] - size[1]) / 2, img.size[0], (img.size[1] + size[1]) / 2)
+			img = img.crop(box)
+
+		# Is more landscape than target, scale and crop
+		elif desired_ratio < img_ratio:
+			img = img.resize((int(size[1] * img.size[0] / img.size[1]), int(size[1])),
+					Image.BICUBIC)
+			box = ((img.size[0] - size[0]) / 2, 0, (img.size[0] + size[0]) / 2, img.size[1])
+			img = img.crop(box)
+
+		# Is exactly the same ratio as target
+		else:
+			img = img.resize(size, Image.BICUBIC)
+
+		if modified_path.endswith(".jpg") and img.mode != "RGB":
+			img = img.convert("RGB")
+
+		img.save(modified_path, lossless=True)
+
+
+def find_source_file(img):
+	upload_dir = current_app.config["UPLOAD_DIR"]
+	source_filepath = os.path.join(upload_dir, img)
+	if os.path.isfile(source_filepath):
+		return source_filepath
+
+	period = source_filepath.rfind(".")
+	start = source_filepath[:period]
+	ext = source_filepath[period + 1:]
+	if ext not in ALLOWED_EXTENSIONS:
 		abort(404)
 
-	# Get current and desired ratio for the images
-	img_ratio = img.size[0] / float(img.size[1])
-	ratio = size[0] / float(size[1])
+	for other_ext in ALLOWED_EXTENSIONS:
+		other_path = f"{start}.{other_ext}"
+		if ext != other_ext and os.path.isfile(other_path):
+			return other_path
 
-	# Is more portrait than target, scale and crop
-	if ratio > img_ratio:
-		img = img.resize((int(size[0]), int(size[0] * img.size[1] / img.size[0])),
-				Image.BICUBIC)
-		box = (0, (img.size[1] - size[1]) / 2, img.size[0], (img.size[1] + size[1]) / 2)
-		img = img.crop(box)
-
-	# Is more landscape than target, scale and crop
-	elif ratio < img_ratio:
-		img = img.resize((int(size[1] * img.size[0] / img.size[1]), int(size[1])),
-				Image.BICUBIC)
-		box = ((img.size[0] - size[0]) / 2, 0, (img.size[0] + size[0]) / 2, img.size[1])
-		img = img.crop(box)
-
-	# Is exactly the same ratio as target
-	else:
-		img = img.resize(size, Image.BICUBIC)
-
-	img.save(modified_path)
+	abort(404)
 
 
 @bp.route("/thumbnails/<int:level>/<img>")
@@ -70,15 +93,40 @@ def make_thumbnail(img, level):
 		abort(403)
 	w, h = ALLOWED_RESOLUTIONS[level - 1]
 
-	upload_dir = current_app.config["UPLOAD_DIR"]
 	thumbnail_dir = current_app.config["THUMBNAIL_DIR"]
 	mkdir(thumbnail_dir)
 
 	output_dir = os.path.join(thumbnail_dir, str(level))
 	mkdir(output_dir)
 
-	cache_filepath  = os.path.join(output_dir, img)
-	source_filepath = os.path.join(upload_dir, img)
+	cache_filepath = os.path.join(output_dir, img)
+	if not os.path.isfile(cache_filepath):
+		source_filepath = find_source_file(img)
+		resize_and_crop(source_filepath, cache_filepath, (w, h))
 
-	resize_and_crop(source_filepath, cache_filepath, (w, h))
-	return send_file(cache_filepath)
+	res = send_file(cache_filepath)
+	res.headers["Cache-Control"] = "max-age=604800" # 1 week
+	return res
+
+
+@bp.route("/thumbnails/youtube/<id_>.jpg")
+def youtube(id_: str):
+	if not re.match(r"^[A-Za-z0-9\-_]+$", id_):
+		abort(400)
+
+	cache_dir = os.path.join(current_app.config["THUMBNAIL_DIR"], "youtube")
+	os.makedirs(cache_dir, exist_ok=True)
+	cache_filepath = os.path.join(cache_dir, id_ + ".jpg")
+
+	url = f"https://img.youtube.com/vi/{id_}/default.jpg"
+
+	response = requests.get(url, stream=True)
+	if response.status_code != 200:
+		abort(response.status_code)
+
+	with open(cache_filepath, "wb") as file:
+		file.write(response.content)
+
+	res = send_file(cache_filepath)
+	res.headers["Cache-Control"] = "max-age=604800"  # 1 week
+	return res

app/blueprints/todo/editor.py

@@ -17,12 +17,12 @@
 
 from flask import redirect, url_for, abort, render_template, request
 from flask_login import current_user, login_required
-from sqlalchemy import or_
+from sqlalchemy import or_, and_
 
 from app.models import Package, PackageState, PackageScreenshot, PackageUpdateConfig, ForumTopic, db, \
 	PackageRelease, Permission, UserRank, License, MetaPackage, Dependency, AuditLogEntry, Tag, MinetestRelease
 from app.querybuilder import QueryBuilder
-from app.utils import get_int_or_abort, is_yes
+from app.utils import get_int_or_abort, is_yes, rank_required
 from . import bp
 
 
@@ -44,7 +44,7 @@ def view_editor():
 
 	releases = None
 	if can_approve_rel:
-		releases = PackageRelease.query.filter_by(approved=False).all()
+		releases = PackageRelease.query.filter_by(approved=False, task_id=None).all()
 
 	screenshots = None
 	if can_approve_scn:
@@ -90,42 +90,10 @@ def view_editor():
 			unfulfilled_meta_packages=unfulfilled_meta_packages, audit_log=audit_log)
 
 
-@bp.route("/todo/topics/")
-@login_required
-def topics():
-	qb = QueryBuilder(request.args)
-	qb.set_sort_if_none("date")
-	query = qb.build_topic_query()
-
-	tmp_q = ForumTopic.query
-	if not qb.show_discarded:
-		tmp_q = tmp_q.filter_by(discarded=False)
-	total = tmp_q.count()
-	topic_count = query.count()
-
-	page = get_int_or_abort(request.args.get("page"), 1)
-	num = get_int_or_abort(request.args.get("n"), 100)
-	if num > 100 and not current_user.rank.at_least(UserRank.APPROVER):
-		num = 100
-
-	query = query.paginate(page=page, per_page=num)
-	next_url = url_for("todo.topics", page=query.next_num, query=qb.search,
-		show_discarded=qb.show_discarded, n=num, sort=qb.order_by) \
-		if query.has_next else None
-	prev_url = url_for("todo.topics", page=query.prev_num, query=qb.search,
-		show_discarded=qb.show_discarded, n=num, sort=qb.order_by) \
-		if query.has_prev else None
-
-	return render_template("todo/topics.html", current_tab="topics", topics=query.items, total=total,
-		topic_count=topic_count, query=qb.search, show_discarded=qb.show_discarded,
-		next_url=next_url, prev_url=prev_url, page=page, page_max=query.pages,
-		n=num, sort_by=qb.order_by)
-
-
 @bp.route("/todo/tags/")
 @login_required
 def tags():
-	qb    = QueryBuilder(request.args)
+	qb    = QueryBuilder(request.args, cookies=True)
 	qb.set_sort_if_none("score", "desc")
 	query = qb.build_package_query()
 
@@ -220,3 +188,28 @@ def mtver_support():
 
 	return render_template("todo/mtver_support.html", current_tab="screenshots",
 			packages=query.all(), sort_by=sort_by, is_mtm_only=is_mtm_only, current_stable=current_stable)
+
+
+@bp.route("/todo/topics/mismatch/")
+@rank_required(UserRank.EDITOR)
+def topics_mismatch():
+	missing_topics = Package.query.filter(Package.forums.is_not(None)) .filter(~ForumTopic.query.filter(ForumTopic.topic_id == Package.forums).exists()).all()
+
+	packages_bad_author = (
+		db.session.query(Package, ForumTopic)
+		.select_from(Package)
+		.join(ForumTopic, Package.forums == ForumTopic.topic_id)
+		.filter(Package.author_id != ForumTopic.author_id)
+		.all())
+
+	packages_bad_title = (
+		db.session.query(Package, ForumTopic)
+		.select_from(Package)
+		.join(ForumTopic, Package.forums == ForumTopic.topic_id)
+		.filter(and_(ForumTopic.name != Package.name, ~ForumTopic.title.ilike("%" + Package.title + "%"), ~ForumTopic.title.ilike("%" + Package.name + "%")))
+		.all())
+
+	return render_template("todo/topics_mismatch.html",
+			missing_topics=missing_topics,
+			packages_bad_author=packages_bad_author,
+			packages_bad_title=packages_bad_title)

app/blueprints/todo/user.py

@@ -113,11 +113,12 @@ def apply_all_updates(username):
 				PackageRelease.commit_hash == package.update_config.last_commit)).count() > 0:
 			continue
 
-		title = package.update_config.get_title()
+		title = package.update_config.title
 		ref = package.update_config.get_ref()
 
 		rel = PackageRelease()
 		rel.package = package
+		rel.name = title
 		rel.title = title
 		rel.url = ""
 		rel.task_id = uuid()

app/blueprints/translate/__init__.py

@@ -0,0 +1,48 @@
+# ContentDB
+# Copyright (C) 2024 rubenwardy
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program.  If not, see <https://www.gnu.org/licenses/>.
+
+
+from flask import Blueprint, render_template, request
+from sqlalchemy import or_
+
+from app.models import Package, PackageState, db, PackageTranslation
+
+bp = Blueprint("translate", __name__)
+
+
+@bp.route("/translate/")
+def translate():
+	query = Package.query.filter(
+			Package.state == PackageState.APPROVED,
+			or_(
+				Package.translation_url.is_not(None),
+				Package.translations.any(PackageTranslation.language_id != "en")
+			))
+
+	has_langs = request.args.getlist("has_lang")
+	for lang in has_langs:
+		query = query.filter(Package.translations.any(PackageTranslation.language_id == lang))
+
+	not_langs = request.args.getlist("not_lang")
+	for lang in not_langs:
+		query = query.filter(~Package.translations.any(PackageTranslation.language_id == lang))
+
+	supports_translation = (query
+			.order_by(Package.translation_url.is_(None), db.desc(Package.score))
+			.all())
+
+	return render_template("translate/index.html",
+			supports_translation=supports_translation, has_langs=has_langs, not_langs=not_langs)

app/blueprints/users/account.py

@@ -16,7 +16,7 @@
 
 import datetime
 
-from flask import redirect, abort, render_template, flash, request, url_for
+from flask import redirect, abort, render_template, flash, request, url_for, Response
 from flask_babel import gettext, get_locale, lazy_gettext
 from flask_login import current_user, login_required, logout_user, login_user
 from flask_wtf import FlaskForm
@@ -26,10 +26,10 @@ from wtforms.validators import InputRequired, Length, Regexp, DataRequired, Opti
 
 from app.tasks.emails import send_verify_email, send_anon_email, send_unsubscribe_verify, send_user_email
 from app.utils import random_string, make_flask_login_password, is_safe_url, check_password_hash, add_audit_log, \
-	nonempty_or_none, post_login, is_username_valid
+	nonempty_or_none, post_login
 from . import bp
-from app.models import User, AuditSeverity, db, UserRank, PackageAlias, EmailSubscription, UserNotificationPreferences, \
-	UserEmailVerification
+from app.models import User, AuditSeverity, db, EmailSubscription, UserEmailVerification
+from app.logic.users import create_user
 
 
 class LoginForm(FlaskForm):
@@ -113,46 +113,13 @@ def handle_register(form):
 		flash(gettext("Incorrect captcha answer"), "danger")
 		return
 
-	if not is_username_valid(form.username.data):
-		flash(gettext("Username is invalid"))
+	user = create_user(form.username.data, form.display_name.data, form.email.data)
+	if isinstance(user, Response):
+		return user
+	elif user is None:
 		return
 
-	user_by_name = User.query.filter(or_(
-			User.username == form.username.data,
-			User.username == form.display_name.data,
-			User.display_name == form.display_name.data,
-			User.forums_username == form.username.data,
-			User.github_username == form.username.data)).first()
-	if user_by_name:
-		if user_by_name.rank == UserRank.NOT_JOINED and user_by_name.forums_username:
-			flash(gettext("An account already exists for that username but hasn't been claimed yet."), "danger")
-			return redirect(url_for("users.claim_forums", username=user_by_name.forums_username))
-		else:
-			flash(gettext("That username/display name is already in use, please choose another."), "danger")
-			return
-
-	alias_by_name = PackageAlias.query.filter(or_(
-			PackageAlias.author==form.username.data,
-			PackageAlias.author==form.display_name.data)).first()
-	if alias_by_name:
-		flash(gettext("That username/display name is already in use, please choose another."), "danger")
-		return
-
-	user_by_email = User.query.filter_by(email=form.email.data).first()
-	if user_by_email:
-		send_anon_email.delay(form.email.data, get_locale().language, gettext("Email already in use"),
-			gettext("We were unable to create the account as the email is already in use by %(display_name)s. Try a different email address.",
-					display_name=user_by_email.display_name))
-		return redirect(url_for("users.email_sent"))
-	elif EmailSubscription.query.filter_by(email=form.email.data, blacklisted=True).count() > 0:
-		flash(gettext("That email address has been unsubscribed/blacklisted, and cannot be used"), "danger")
-		return
-
-	user = User(form.username.data, False, form.email.data, make_flask_login_password(form.password.data))
-	user.notification_preferences = UserNotificationPreferences(user)
-	if form.display_name.data:
-		user.display_name = form.display_name.data
-	db.session.add(user)
+	user.password = make_flask_login_password(form.password.data)
 
 	add_audit_log(AuditSeverity.USER, user, "Registered with email, display name=" + user.display_name,
 				  url_for("users.profile", username=user.username))
@@ -319,9 +286,7 @@ def verify_email():
 		flash(gettext("Unknown verification token!"), "danger")
 		return redirect(url_for("homepage.home"))
 
-	delta = (datetime.datetime.now() - ver.created_at)
-	delta: datetime.timedelta
-	if delta.total_seconds() > 12*60*60:
+	if ver.is_expired:
 		flash(gettext("Token has expired"), "danger")
 		db.session.delete(ver)
 		db.session.commit()

app/blueprints/users/claim.py

@@ -15,11 +15,12 @@
 # along with this program.  If not, see <https://www.gnu.org/licenses/>.
 
 from flask_babel import gettext
+from flask_login import current_user
 
 from . import bp
 from flask import redirect, render_template, session, request, flash, url_for
 from app.models import db, User, UserRank
-from app.utils import random_string, login_user_set_active, is_username_valid
+from app.utils import random_string, login_user_set_active
 from app.tasks.forumtasks import check_forum_account
 from app.utils.phpbbparser import get_profile
 
@@ -31,26 +32,25 @@ def claim():
 
 @bp.route("/user/claim-forums/", methods=["GET", "POST"])
 def claim_forums():
+	if current_user.is_authenticated:
+		return redirect(url_for("homepage.home"))
+
 	username = request.args.get("username")
 	if username is None:
 		username = ""
 	else:
 		method = request.args.get("method")
 
-		if not is_username_valid(username):
-			flash(gettext("Invalid username, Only alphabetic letters (A-Za-z), numbers (0-9), underscores (_), minuses (-), and periods (.) allowed. Consider contacting an admin"), "danger")
-			return redirect(url_for("users.claim_forums"))
-
 		user = User.query.filter_by(forums_username=username).first()
 		if user and user.rank.at_least(UserRank.NEW_MEMBER):
 			flash(gettext("User has already been claimed"), "danger")
 			return redirect(url_for("users.claim_forums"))
 		elif method == "github":
 			if user is None or user.github_username is None:
-				flash(gettext("Unable to get GitHub username for user"), "danger")
+				flash(gettext("Unable to get GitHub username for user. Make sure the forum account exists."), "danger")
 				return redirect(url_for("users.claim_forums", username=username))
 			else:
-				return redirect(url_for("github.start"))
+				return redirect(url_for("vcs.github_start"))
 
 	if "forum_token" in session:
 		token = session["forum_token"]
@@ -62,9 +62,11 @@ def claim_forums():
 		ctype = request.form.get("claim_type")
 		username = request.form.get("username")
 
-		if not is_username_valid(username):
-			flash(gettext("Invalid username, Only alphabetic letters (A-Za-z), numbers (0-9), underscores (_), minuses (-), and periods (.) allowed. Consider contacting an admin"), "danger")
-		elif ctype == "github":
+		if User.query.filter(User.username == username, User.forums_username.is_(None)).first():
+			flash(gettext("A ContentDB user with that name already exists. Please contact an admin to link to your forum account"), "danger")
+			return redirect(url_for("users.claim_forums"))
+
+		if ctype == "github":
 			task = check_forum_account.delay(username)
 			return redirect(url_for("tasks.check", id=task.id, r=url_for("users.claim_forums", username=username, method="github")))
 		elif ctype == "forum":

app/blueprints/users/profile.py

@@ -22,7 +22,7 @@ from flask_babel import gettext
 from flask_login import current_user, login_required
 from sqlalchemy import func, text
 
-from app.models import User, db, Package, PackageReview, PackageState, PackageType, UserRank
+from app.models import User, db, Package, PackageReview, PackageState, PackageType, UserRank, Collection
 from app.utils import get_daterange_options
 from app.tasks.forumtasks import check_forum_account
 
@@ -162,10 +162,7 @@ def get_user_medals(user: User) -> Tuple[List[Medal], List[Medal]]:
 	if user_package_ranks:
 		top_rank = user_package_ranks[2]
 		top_type = PackageType.coerce(user_package_ranks[0])
-		if top_rank == 1:
-			title = gettext(u"Top %(type)s", type=top_type.text.lower())
-		else:
-			title = gettext(u"Top %(group)d %(type)s", group=top_rank, type=top_type.text.lower())
+		title = top_type.get_top_ordinal(top_rank)
 		if top_type == PackageType.MOD:
 			icon = "fa-box"
 		elif top_type == PackageType.GAME:
@@ -173,8 +170,7 @@ def get_user_medals(user: User) -> Tuple[List[Medal], List[Medal]]:
 		else:
 			icon = "fa-paint-brush"
 
-		description = gettext(u"%(display_name)s has a %(type)s placed at #%(place)d.",
-				display_name=user.display_name, type=top_type.text.lower(), place=top_rank)
+		description = top_type.get_top_ordinal_description(user.display_name, top_rank)
 		unlocked.append(
 				Medal.make_unlocked(place_to_color(top_rank), icon, title, description))
 
@@ -230,11 +226,14 @@ def profile(username):
 		.filter(Package.author != user) \
 		.order_by(db.asc(Package.title)).all()
 
+	pinned_collections = user.collections.filter(Collection.private == False,
+			Collection.pinned == True, Collection.packages.any()).all()
+
 	unlocked, locked = get_user_medals(user)
 	# Process GET or invalid POST
 	return render_template("users/profile.html", user=user,
 			packages=packages, maintained_packages=maintained_packages,
-			medals_unlocked=unlocked, medals_locked=locked)
+			medals_unlocked=unlocked, medals_locked=locked, pinned_collections=pinned_collections)
 
 
 @bp.route("/users/<username>/check-forums/", methods=["POST"])
@@ -256,6 +255,18 @@ def user_check_forums(username):
 	return redirect(url_for("tasks.check", id=task.id, r=next_url))
 
 
+@bp.route("/users/<username>/remove-profile-pic/", methods=["POST"])
+@login_required
+def user_remove_profile_pic(username):
+	user = User.query.filter_by(username=username).one_or_404()
+	if current_user != user and not current_user.rank.at_least(UserRank.MODERATOR):
+		abort(403)
+
+	user.profile_pic = None
+	db.session.commit()
+	return redirect(url_for("users.profile_edit", username=username))
+
+
 @bp.route("/user/stats/")
 @login_required
 def statistics_redirect():

app/blueprints/users/settings.py

@@ -18,6 +18,7 @@ from flask import redirect, abort, render_template, request, flash, url_for
 from flask_babel import gettext, get_locale, lazy_gettext
 from flask_login import current_user, login_required, logout_user
 from flask_wtf import FlaskForm
+from kombu import uuid
 from sqlalchemy import or_
 from wtforms import StringField, SubmitField, BooleanField, SelectField
 from wtforms.validators import Length, Optional, Email, URL
@@ -25,6 +26,7 @@ from wtforms.validators import Length, Optional, Email, URL
 from app.models import User, AuditSeverity, db, UserRank, PackageAlias, EmailSubscription, UserNotificationPreferences, \
 	UserEmailVerification, Permission, NotificationType, UserBan
 from app.tasks.emails import send_verify_email
+from app.tasks.usertasks import update_github_user_id
 from app.utils import nonempty_or_none, add_audit_log, random_string, rank_required, has_blocked_domains
 from . import bp
 
@@ -128,8 +130,7 @@ def profile_edit(username):
 		abort(404)
 
 	if not user.can_see_edit_profile(current_user):
-		flash(gettext("Permission denied"), "danger")
-		return redirect(url_for("users.profile", username=username))
+		abort(403)
 
 	form = UserProfileForm(obj=user)
 	if form.validate_on_submit():
@@ -242,9 +243,31 @@ def account(username):
 	if not user:
 		abort(404)
 
+	if not user.can_see_edit_profile(current_user):
+		abort(403)
+
 	return render_template("users/account.html", user=user, tabs=get_setting_tabs(user), current_tab="account")
 
 
+@bp.route("/users/<username>/settings/account/disconnect-github/", methods=["POST"])
+def disconnect_github(username: str):
+	user: User = User.query.filter_by(username=username).one_or_404()
+
+	if not user.can_see_edit_profile(current_user):
+		abort(403)
+
+	if user.password and user.email:
+		user.github_user_id = None
+		user.github_username = None
+		db.session.commit()
+
+		flash(gettext("Removed GitHub account"), "success")
+	else:
+		flash(gettext("You need to add an email address and password before you can remove your GitHub account"), "danger")
+
+	return redirect(url_for("users.account", username=username))
+
+
 @bp.route("/users/<username>/delete/", methods=["GET", "POST"])
 @rank_required(UserRank.ADMIN)
 def delete(username):
@@ -275,6 +298,9 @@ def delete(username):
 			db.session.delete(reply)
 		for thread in user.threads.all():
 			db.session.delete(thread)
+		for token in user.tokens.all():
+			db.session.delete(token)
+		user.profile_pic = None
 		user.email = None
 
 		if user.rank != UserRank.BANNED:
@@ -320,6 +346,8 @@ def modtools(username):
 		add_audit_log(severity, current_user, "Edited {}'s account".format(user.display_name),
 					  url_for("users.profile", username=username))
 
+		redirect_target = url_for("users.modtools", username=username)
+
 		# Copy form fields to user_profile fields
 		if user.check_perm(current_user, Permission.CHANGE_USERNAMES):
 			if user.username != form.username.data:
@@ -332,14 +360,21 @@ def modtools(username):
 
 			user.display_name = form.display_name.data
 			user.forums_username = nonempty_or_none(form.forums_username.data)
-			user.github_username = nonempty_or_none(form.github_username.data)
+			github_username = nonempty_or_none(form.github_username.data)
+			if github_username is None:
+				user.github_username = None
+				user.github_user_id = None
+			else:
+				task_id = uuid()
+				update_github_user_id.apply_async((user.id, github_username), task_id=task_id)
+				redirect_target = url_for("tasks.check", id=task_id, r=redirect_target)
 
 		if user.check_perm(current_user, Permission.CHANGE_RANK):
-			new_rank = form["rank"].data
+			new_rank = form.rank.data
 			if current_user.rank.at_least(new_rank):
 				if new_rank != user.rank:
-					user.rank = form["rank"].data
-					msg = "Set rank of {} to {}".format(user.display_name, user.rank.get_title())
+					user.rank = form.rank.data
+					msg = "Set rank of {} to {}".format(user.display_name, user.rank.title)
 					add_audit_log(AuditSeverity.MODERATION, current_user, msg,
 								  url_for("users.profile", username=username))
 			else:
@@ -347,7 +382,7 @@ def modtools(username):
 
 		db.session.commit()
 
-		return redirect(url_for("users.modtools", username=username))
+		return redirect(redirect_target)
 
 	return render_template("users/modtools.html", user=user, form=form, tabs=get_setting_tabs(user), current_tab="modtools")
 

app/blueprints/vcs/__init__.py

@@ -1,5 +1,5 @@
 # ContentDB
-# Copyright (C) rubenwardy
+# Copyright (C) 2024 rubenwardy
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU Affero General Public License as published by
@@ -14,8 +14,9 @@
 # You should have received a copy of the GNU Affero General Public License
 # along with this program.  If not, see <https://www.gnu.org/licenses/>.
 
-import user_agents
 
+from flask import Blueprint
 
-def test_minetest_is_not_bot():
-	assert not user_agents.parse("Minetest/5.5.1 (Linux/4.14.193+-ab49821 aarch64)").is_bot
+bp = Blueprint("vcs", __name__)
+
+from . import github, gitlab

app/blueprints/vcs/common.py

@@ -0,0 +1,42 @@
+# ContentDB
+# Copyright (C) 2024  rubenwardy
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program.  If not, see <https://www.gnu.org/licenses/>.
+
+
+from app.blueprints.api.support import error
+from app.models import Package, APIToken, Permission, PackageState
+
+
+def get_packages_for_vcs_and_token(token: APIToken, repo_url: str) -> list[Package]:
+	if token.package:
+		packages = [token.package]
+		if not token.package.check_perm(token.owner, Permission.APPROVE_RELEASE):
+			return error(403, "You do not have the permission to approve releases")
+
+		actual_repo_url: str = token.package.repo or ""
+		if repo_url not in actual_repo_url.lower():
+			return error(400, "Repo URL does not match the API token's package")
+	else:
+		# Get package
+		packages = Package.query.filter(
+			Package.repo.ilike("%{}%".format(repo_url)), Package.state != PackageState.DELETED).all()
+		if len(packages) == 0:
+			return error(400,
+					"Could not find package, did you set the VCS repo in CDB correctly? Expected {}".format(repo_url))
+		packages = [x for x in packages if x.check_perm(token.owner, Permission.APPROVE_RELEASE)]
+		if len(packages) == 0:
+			return error(403, "You do not have the permission to approve releases")
+
+	return packages

app/blueprints/vcs/github.py

@@ -0,0 +1,200 @@
+# ContentDB
+# Copyright (C) 2018-24 rubenwardy
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program.  If not, see <https://www.gnu.org/licenses/>.
+
+import datetime
+import hmac
+
+import requests
+from flask import abort, Response
+from flask import redirect, url_for, request, flash, jsonify, current_app
+from flask_babel import gettext
+from flask_login import current_user
+
+from app import github, csrf
+from app.blueprints.api.support import error, api_create_vcs_release
+from app.logic.users import create_user
+from app.models import db, User, APIToken, AuditSeverity
+from app.utils import abs_url_for, add_audit_log, login_user_set_active, is_safe_url
+
+from . import bp
+from .common import get_packages_for_vcs_and_token
+
+
+@bp.route("/github/start/")
+def github_start():
+	next = request.args.get("next")
+	if next and not is_safe_url(next):
+		abort(400)
+
+	return github.authorize("", redirect_uri=abs_url_for("vcs.github_callback", next=next))
+
+
+@bp.route("/github/view/")
+def github_view_permissions():
+	url = "https://github.com/settings/connections/applications/" + \
+			current_app.config["GITHUB_CLIENT_ID"]
+	return redirect(url)
+
+
+@bp.route("/github/callback/")
+@github.authorized_handler
+def github_callback(oauth_token):
+	if oauth_token is None:
+		flash(gettext("Authorization failed [err=gh-oauth-login-failed]"), "danger")
+		return redirect(url_for("users.login"))
+
+	next = request.args.get("next")
+	if next and not is_safe_url(next):
+		abort(400)
+
+	redirect_to = next
+	if redirect_to is None:
+		redirect_to = url_for("homepage.home")
+
+	# Get GitGub username
+	url = "https://api.github.com/user"
+	r = requests.get(url, headers={"Authorization": "token " + oauth_token})
+	json = r.json()
+	user_id = json["id"]
+	github_username = json["login"]
+	if type(user_id) is not int:
+		abort(400)
+
+	# Get user by GitHub user ID
+	user_by_github = User.query.filter(User.github_user_id == user_id).one_or_none()
+
+	# If logged in, connect
+	if current_user and current_user.is_authenticated:
+		if user_by_github is None:
+			current_user.github_username = github_username
+			current_user.github_user_id = user_id
+			db.session.commit()
+			flash(gettext("Linked GitHub to account"), "success")
+			return redirect(redirect_to)
+		elif user_by_github == current_user:
+			return redirect(redirect_to)
+		else:
+			flash(gettext("GitHub account is already associated with another user: %(username)s",
+					username=user_by_github.username), "danger")
+			return redirect(redirect_to)
+
+	# Log in to existing account
+	elif user_by_github:
+		ret = login_user_set_active(user_by_github, next, remember=True)
+		if ret is None:
+			flash(gettext("Authorization failed [err=gh-login-failed]"), "danger")
+			return redirect(url_for("users.login"))
+
+		add_audit_log(AuditSeverity.USER, user_by_github, "Logged in using GitHub OAuth",
+				url_for("users.profile", username=user_by_github.username))
+		db.session.commit()
+		return ret
+
+	# Sign up
+	else:
+		user = create_user(github_username, github_username, None, "GitHub")
+		if isinstance(user, Response):
+			return user
+		elif user is None:
+			return redirect(url_for("users.login"))
+
+		user.github_username = github_username
+		user.github_user_id = user_id
+
+		add_audit_log(AuditSeverity.USER, user, "Registered with GitHub, display name=" + user.display_name,
+				url_for("users.profile", username=user.username))
+
+		db.session.commit()
+
+		ret = login_user_set_active(user, next, remember=True)
+		if ret is None:
+			flash(gettext("Authorization failed [err=gh-login-failed]"), "danger")
+			return redirect(url_for("users.login"))
+
+		return ret
+
+
+def _find_api_token(header_signature: str) -> APIToken:
+	sha_name, signature = header_signature.split('=')
+	if sha_name != 'sha1':
+		error(403, "Expected SHA1 payload signature")
+
+	for token in APIToken.query.all():
+		mac = hmac.new(token.access_token.encode("utf-8"), msg=request.data, digestmod='sha1')
+
+		if hmac.compare_digest(str(mac.hexdigest()), signature):
+			return token
+
+	error(401, "Invalid authentication, couldn't validate API token")
+
+
+@bp.route("/github/webhook/", methods=["POST"])
+@csrf.exempt
+def github_webhook():
+	json = request.json
+
+	header_signature = request.headers.get('X-Hub-Signature')
+	if header_signature is None:
+		return error(403, "Expected payload signature")
+
+	token = _find_api_token(header_signature)
+	packages = get_packages_for_vcs_and_token(token, "github.com/" + json["repository"]["full_name"])
+
+	for package in packages:
+		#
+		# Check event
+		#
+		event = request.headers.get("X-GitHub-Event")
+		if event == "push":
+			ref = json["after"]
+			title = datetime.datetime.utcnow().strftime("%Y-%m-%d") + " " + ref[:5]
+			branch = json["ref"].replace("refs/heads/", "")
+			if package.update_config and package.update_config.ref:
+				if branch != package.update_config.ref:
+					continue
+			elif branch not in ["master", "main"]:
+				continue
+
+		elif event == "create":
+			ref_type = json.get("ref_type")
+			if ref_type != "tag":
+				return jsonify({
+					"success": False,
+					"message": "Webhook ignored, as it's a non-tag create event. ref_type='{}'.".format(ref_type)
+				})
+
+			ref = json["ref"]
+			title = ref
+
+		elif event == "ping":
+			return jsonify({"success": True, "message": "Ping successful"})
+
+		else:
+			return error(400, "Unsupported event: '{}'. Only 'push', 'create:tag', and 'ping' are supported."
+					.format(event or "null"))
+
+		#
+		# Perform release
+		#
+		if package.releases.filter_by(commit_hash=ref).count() > 0:
+			return
+
+		return api_create_vcs_release(token, package, title, title, None, ref, reason="Webhook")
+
+	return jsonify({
+		"success": False,
+		"message": "No release made. Either the release already exists or the event was filtered based on the branch"
+	})

app/blueprints/vcs/gitlab.py

@@ -0,0 +1,86 @@
+# ContentDB
+# Copyright (C) 2020-24  rubenwardy
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program.  If not, see <https://www.gnu.org/licenses/>.
+
+import datetime
+
+from flask import request, jsonify
+
+from app import csrf
+from app.blueprints.api.support import error, api_create_vcs_release
+from app.models import APIToken
+
+from . import bp
+from .common import get_packages_for_vcs_and_token
+
+
+def webhook_impl():
+	json = request.json
+
+	# Get all tokens for package
+	secret = request.headers.get("X-Gitlab-Token")
+	if secret is None:
+		return error(403, "Token required")
+
+	token: APIToken = APIToken.query.filter_by(access_token=secret).first()
+	if token is None:
+		return error(403, "Invalid authentication")
+
+	packages = get_packages_for_vcs_and_token(token, json["project"]["web_url"].replace("https://", "").replace("http://", ""))
+	for package in packages:
+		#
+		# Check event
+		#
+		event = json["event_name"]
+		if event == "push":
+			ref = json["after"]
+			title = datetime.datetime.utcnow().strftime("%Y-%m-%d") + " " + ref[:5]
+			branch = json["ref"].replace("refs/heads/", "")
+			if package.update_config and package.update_config.ref:
+				if branch != package.update_config.ref:
+					continue
+			elif branch not in ["master", "main"]:
+				continue
+
+		elif event == "tag_push":
+			ref = json["ref"]
+			title = ref.replace("refs/tags/", "")
+
+		else:
+			return error(400, "Unsupported event: '{}'. Only 'push', 'create:tag', and 'ping' are supported."
+					.format(event or "null"))
+
+		#
+		# Perform release
+		#
+		if package.releases.filter_by(commit_hash=ref).count() > 0:
+			continue
+
+		return api_create_vcs_release(token, package, title, title, None, ref, reason="Webhook")
+
+	return jsonify({
+		"success": False,
+		"message": "No release made. Either the release already exists or the event was filtered based on the branch"
+	})
+
+
+
+@bp.route("/gitlab/webhook/", methods=["POST"])
+@csrf.exempt
+def gitlab_webhook():
+	try:
+		return webhook_impl()
+	except KeyError as err:
+		return error(400, "Missing field: {}".format(err.args[0]))

app/default_data.py

@@ -105,6 +105,7 @@ def populate_test_data(session):
 
 	rel = PackageRelease()
 	rel.package = mod
+	rel.name = "v1.0.0"
 	rel.title = "v1.0.0"
 	rel.url = "https://github.com/ezhh/handholds/archive/master.zip"
 	rel.approved = True
@@ -142,6 +143,7 @@ awards.register_achievement("award_mesefind",{
 	rel = PackageRelease()
 	rel.package = mod1
 	rel.min_rel = v51
+	rel.name = "v1.0.0"
 	rel.title = "v1.0.0"
 	rel.url = "https://github.com/rubenwardy/awards/archive/master.zip"
 	rel.approved = True
@@ -254,6 +256,7 @@ No warranty is provided, express or implied, for any part of the project.
 
 	rel = PackageRelease()
 	rel.package = mod
+	rel.name = "v1.0.0"
 	rel.title = "v1.0.0"
 	rel.max_rel = v4
 	rel.url = "https://github.com/ezhh/handholds/archive/master.zip"
@@ -367,6 +370,7 @@ Uses the CTF PvP Engine.
 
 	rel = PackageRelease()
 	rel.package = game1
+	rel.name = "v1.0.0"
 	rel.title = "v1.0.0"
 	rel.url = "https://github.com/rubenwardy/capturetheflag/archive/master.zip"
 	rel.approved = True
@@ -388,6 +392,7 @@ Uses the CTF PvP Engine.
 
 	rel = PackageRelease()
 	rel.package = mod
+	rel.name = "v1.0.0"
 	rel.title = "v1.0.0"
 	rel.url = "http://mamadou3.free.fr/Minetest/PixelBOX.zip"
 	rel.approved = True

app/flatpages/about.md

@@ -12,8 +12,10 @@ ContentDB is open source software, licensed under AGPLv3.0.
 
 <a href="https://github.com/minetest/contentdb/" class="btn btn-primary me-1">Source code</a>
 <a href="https://github.com/minetest/contentdb/issues/" class="btn btn-secondary me-1">Issue tracker</a>
-<a href="https://rubenwardy.com/contact/" class="btn btn-secondary me-1">Contact admin</a>
-<a href="https://monitor.rubenwardy.com/d/3ELzFy3Wz/contentdb" class="btn btn-secondary">Stats / monitoring</a>
+<a href="{{ admin_contact_url }}" class="btn btn-secondary me-1">Contact admin</a>
+{% if monitoring_url -%}
+<a href="{{ monitoring_url }}" class="btn btn-secondary">Stats / monitoring</a>
+{%- endif %}
 
 ## Why was ContentDB created?
 
@@ -31,9 +33,17 @@ You should read
 [the official Minetest Modding Book](https://rubenwardy.com/minetest_modding_book/)
 for a guide to making mods and games using Minetest.
 
-## How can I support / donate to ContentDB?
 
-You can donate to rubenwardy to cover ContentDB's costs and support future
-development.
+<h2 id="donate">How can I support / donate to ContentDB?</h2>
+
+You can donate to rubenwardy to cover ContentDB's costs and support future development.
+
+For more information about the cost of ContentDB and what rubenwardy does, see his donation page:
 
 <a href="https://rubenwardy.com/donate/" class="btn btn-primary me-1">Donate</a>
+<a href="/donate/" class="btn btn-secondary">Support Creators</a>
+
+## Sponsorships
+
+Minetest and ContentDB are sponsored by <a href="https://sentry.io/" rel="nofollow">sentry.io</a>.
+This provides us with improved error logging and performance insights.

app/flatpages/help.md

@@ -18,6 +18,7 @@ toc: False
 * [Contact Us](contact_us/)
 * [Top Packages Algorithm](top_packages/)
 * [Featured Packages](featured/)
+* [Feeds](feeds/)
 
 ## Help for Package Authors
 
@@ -27,6 +28,8 @@ toc: False
 * [Creating Releases using Webhooks](release_webhooks/)
 * [Package Configuration and Releases Guide](package_config/)
 * [Supported Games](game_support/)
+* [Creating an appealing ContentDB page](appealing_page/)
+
 
 ## Help for Specific User Ranks
 

app/flatpages/help/api.md

@@ -92,19 +92,29 @@ curl -X DELETE https://content.minetest.net/api/delete-token/ \
         * `license`: A [license](#licenses) name.
         * `media_license`: A [license](#licenses) name.
         * `long_description`: Long markdown description.
-        * `repo`: Git repo URL.
+        * `repo`: Source repository (eg: Git)
         * `website`: Website URL.
         * `issue_tracker`: Issue tracker URL.
         * `forums`: forum topic ID.
         * `video_url`: URL to a video.
         * `donate_url`: URL to a donation page.
+        * `translation_url`: URL to send users interested in translating your package.
         * `game_support`: Array of game support information objects. Not currently documented, as subject to change.
+* GET `/api/packages/<username>/<name>/for-client/`
+    * Similar to the read endpoint, but optimised for the Minetest client 
+    * `long_description` is given as a hypertext object, see `/hypertext/` below.
+    * `info_hypertext` is the info sidebar as a hypertext object.
+    * Query arguments
+        * `formspec_version`: Required. See /hypertext/ below.
+        * `include_images`: Optional, defaults to true. If true, images use `<img>`. If false, they're linked.
+        * `protocol_version`: Optional, used to get the correct release.
+        * `engine_version`: Optional, used to get the correct release. Ex: `5.3.0`.
 * GET `/api/packages/<author>/<name>/hypertext/`
     * Converts the long description to [Minetest Markup Language](https://github.com/minetest/minetest/blob/master/doc/lua_api.md#markup-language)
       to be used in a `hypertext` formspec element.
     * Query arguments:
         * `formspec_version`: Required, maximum supported formspec version.
-        * `include_images`: Optional, defaults to true.
+        * `include_images`: Optional, defaults to true. If true, images use `<img>`. If false, they're linked.
     * Returns JSON dictionary with following key:
         * `head`: markup for suggested styling and custom tags, prepend to the body before displaying.
         * `body`: markup for long description.
@@ -177,23 +187,33 @@ Example:
 
     /api/packages/?type=mod&type=game&q=mobs+fun&hide=nonfree&hide=gore
 
-Supported query parameters:
+Filter query parameters:
 
-* `type`:  Package types (`mod`, `game`, `txp`).
+* `type`: Filter by package type (`mod`, `game`, `txp`). Multiple types are OR-ed together.
 * `q`:  Query string.
 * `author`:  Filter by author.
-* `tag`:  Filter by tags.
-* `game`: Filter by [Game Support](/help/game_support/), ex: `Wuzzy/mineclone2`. (experimental, doesn't show items that support every game currently).
-* `random`:  When present, enable random ordering and ignore `sort`.
-* `limit`:  Return at most `limit` packages.
-* `hide`:  Hide content based on [Content Flags](/help/content_flags/).
-* `sort`:  Sort by (`name`, `title`, `score`, `reviews`, `downloads`, `created_at`, `approved_at`, `last_release`).
-* `order`:  Sort ascending (`asc`) or descending (`desc`).
+* `tag`:  Filter by tags. Multiple tags are AND-ed together.
+* `flag`: Filter to show packages with [Content Flags](/help/content_flags/).
+* `hide`:  Hide content based on tags or [Content Flags](/help/content_flags/).
+* `license`: Filter by [license name](#licenses). Multiple licenses are OR-ed together, ie: `&license=MIT&license=LGPL-2.1-only`
+* `game`: Filter by [Game Support](/help/game_support/), ex: `Warr1024/nodecore`. (experimental, doesn't show items that support every game currently).
+* `lang`: Filter by translation support, eg: `en`/`de`/`ja`/`zh_TW`.
 * `protocol_version`:  Only show packages supported by this Minetest protocol version.
 * `engine_version`:  Only show packages supported by this Minetest engine version, eg: `5.3.0`.
+
+Sorting query parameters:
+
+* `sort`:  Sort by (`name`, `title`, `score`, `reviews`, `downloads`, `created_at`, `approved_at`, `last_release`).
+* `order`:  Sort ascending (`asc`) or descending (`desc`).
+* `random`:  When present, enable random ordering and ignore `sort`.
+
+Format query parameters:
+
+* `limit`:  Return at most `limit` packages.
 * `fmt`:  How the response is formatted.
     * `keys`:  author/name only.
     * `short`:  stuff needed for the Minetest client.
+    * `vcs`: `short` but with `repo`.
 
 
 ### Releases
@@ -205,13 +225,16 @@ Supported query parameters:
         * `maintainer`: Filter by maintainer
     * Returns array of release dictionaries with keys:
         * `id`: release ID
+        * `name`: short release name
         * `title`: human-readable title
+        * `release_notes`: string or null, what's new in this release. Markdown.
         * `release_date`: Date released
         * `url`: download URL
         * `commit`: commit hash or null
         * `downloads`: number of downloads
         * `min_minetest_version`: dict or null, minimum supported minetest version (inclusive).
         * `max_minetest_version`: dict or null, minimum supported minetest version (inclusive).
+        * `size`: size of zip file, in bytes.
         * `package`
             * `author`: author username
             * `name`: technical name
@@ -228,6 +251,7 @@ Supported query parameters:
     * Requires authentication.
     * Body can be JSON or multipart form data. Zip uploads must be multipart form data.
     * `title`: human-readable name of the release.
+    * `release_notes`: string or null, what's new in this release.
     * For Git release creation:
         * `method`: must be `git`.
         * `ref`: (Optional) git reference, eg: `master`.
@@ -245,7 +269,13 @@ Examples:
 # Create release from Git
 curl -X POST https://content.minetest.net/api/packages/username/name/releases/new/ \
     -H "Authorization: Bearer YOURTOKEN" -H "Content-Type: application/json" \
-    -d '{ "method": "git", "title": "My Release", "ref": "master" }'
+    -d '{
+        "method": "git",
+        "name": "1.2.3",
+        "title": "My Release",
+        "ref": "master",
+        "release_notes": "some\nrelease\nnotes\n"
+    }'
 
 # Create release from zip upload
 curl -X POST https://content.minetest.net/api/packages/username/name/releases/new/ \
@@ -422,7 +452,6 @@ Supported query parameters:
 * `type`:  Package types (`mod`, `game`, `txp`).
 * `sort`:  Sort by (`name`, `views`, `created_at`).
 * `show_added`:  Show topics that have an existing package.
-* `show_discarded`:  Show topics marked as discarded.
 * `limit`:  Return at most `limit` topics.
 
 
@@ -459,31 +488,43 @@ Supported query parameters:
 
 ### Tags
 
-* GET `/api/tags/` ([View](/api/tags/)):  List of:
-    * `name`:  technical name.
-    * `title`:  human-readable title.
-    * `description`:  tag description or null.
-    * `views`: number of views of this tag.
+* GET `/api/tags/` ([View](/api/tags/))
+    * List of objects with:
+        * `name`:  technical name.
+        * `title`:  human-readable title.
+        * `description`:  tag description or null.
+        * `views`: number of views of this tag.
 
 ### Content Warnings
 
-* GET `/api/content_warnings/` ([View](/api/content_warnings/)):  List of:
-    * `name`:  technical name
-    * `title`:  human-readable title
-    * `description`:  tag description or null
+* GET `/api/content_warnings/` ([View](/api/content_warnings/))
+    * List of objects with 
+        * `name`:  technical name
+        * `title`:  human-readable title
+        * `description`:  tag description or null
 
 ### Licenses
 
-* GET `/api/licenses/` ([View](/api/licenses/)):  List of:
-    * `name`
-    * `is_foss`: whether the license is foss
+* GET `/api/licenses/` ([View](/api/licenses/))
+    * List of objects with: 
+        * `name`
+        * `is_foss`: whether the license is foss
 
 ### Minetest Versions
 
 * GET `/api/minetest_versions/` ([View](/api/minetest_versions/))
-    * `name`: Version name.
-    * `is_dev`: boolean, is dev version.
-    * `protocol_version`: protocol version umber.
+    * List of objects with: 
+        * `name`: Version name.
+        * `is_dev`: boolean, is dev version.
+        * `protocol_version`: protocol version number.
+
+### Languages
+
+* GET `/api/languages/` ([View](/api/languages/))
+    * List of objects with: 
+        * `id`: language code.
+        * `title`: native language name.
+        * `has_contentdb_translation`: whether ContentDB has been translated into this language.
 
 
 ## Misc
@@ -498,6 +539,10 @@ Supported query parameters:
         * `score`: total package score.
         * `score_reviews`: score from reviews.
         * `score_downloads`: score from downloads.
+        * `reviews`: a dictionary of
+          * `positive`: int, number of positive reviews.
+          * `neutral`: int, number of neutral reviews.
+          * `negative`: int, number of negative reviews.
 * GET `/api/homepage/` ([View](/api/homepage/)) - get contents of homepage.
     * `count`:  number of packages
     * `downloads`:  get number of downloads
@@ -519,7 +564,7 @@ Supported query parameters:
     * Content-Type: `text/html` or `text/markdown`.
     * Query arguments:
         * `formspec_version`: Required, maximum supported formspec version. Ie: 6
-        * `include_images`: Optional, defaults to true.
+        * `include_images`: Optional, defaults to true. If true, images use `<img>`. If false, they're linked.
     * Returns JSON dictionary with following key:
         * `head`: markup for suggested styling and custom tags, prepend to the body before displaying.
         * `body`: markup for long description.

app/flatpages/help/appealing_page.md

@@ -0,0 +1,74 @@
+title: Creating an appealing ContentDB page
+
+## Title and short description
+
+Make sure that your package's title is unique, short, and descriptive.
+
+Expand on the title with the short description. You have a limited number
+of characters, use them wisely!
+
+```ini
+# Bad, we know this is a mod for Minetest. Doesn't give much information other than "food"
+description = The food mod for Minetest
+# Much better, says what is actually in this mod!
+description = Adds soup, cakes, bakes and juices
+```
+
+## Thumbnail
+
+A good thumbnail goes a long way to making a package more appealing. It's one of the few things
+a user sees before clicking on your package. Make sure it's possible to tell what a
+thumbnail is when it's small.
+
+For a preview of what your package will look like inside Minetest, see
+Edit Package > Screenshots.
+
+## Screenshots
+
+Upload a good selection of screenshots that show what is possible with your packages.
+You may wish to focus on a different key feature in each of your screenshots.
+
+A lot of users won't bother reading text, and will just look at screenshots.
+
+## Long description
+
+The target audience of your package page is end users.
+The long description should explain what your package is about,
+why the user should choose it, and how to use it if they download it.
+
+[NodeCore](https://content.minetest.net/packages/Warr1024/nodecore/) is a good
+example of what to do. For inspiration, you might want to look at how games on
+Steam write their descriptions.
+
+Your long description might contain:
+
+* What does the package contain/have? ie: list of high-level features.
+* What makes it special? Why should users choose this over another package?
+* How can you use it?
+
+The following are redundant and should probably not be included:
+
+* A heading with the title of the package
+* The short description
+* Links to a Git repository, the forum topic, the package's ContentDB page (ContentDB has fields for this)
+* License (unless you need to give more information than ContentDB's license fields)
+* API reference (unless your mod is a library only)
+* Development instructions for your package (this should be in the repo's README)
+* Screenshots that are already uploaded (unless you want to embed a recipe image in a specific place)
+    * Note: you should avoid images in the long description as they won't be visible inside Minetest,
+      when support for showing the long description is added.
+
+## Localize / Translate your package
+
+According to Google Play, 64% of Minetest Android users don't have English as their main language.
+Adding translation support to your package increases accessibility. Using content translation, you
+can also translate your ContentDB page. See Edit Package > Translation for more information.
+
+<p>
+    <a class="btn btn-primary me-2" href="https://rubenwardy.com/minetest_modding_book/en/quality/translations.html">
+        {{ _("Translation - Minetest Modding Book") }}
+    </a>
+    <a class="btn btn-primary" href="https://api.minetest.net/translations/#translating-content-meta">
+        {{ _("Translating content meta - lua_api.md") }}
+    </a>
+</p>

app/flatpages/help/contact_us.md

@@ -11,4 +11,4 @@ We take copyright violation and other offenses very seriously.
 
 ## Other
 
-<a href="https://rubenwardy.com/contact/" class="btn btn-primary">Contact the admin</a>
+<a href="{{ admin_contact_url }}" class="btn btn-primary">Contact the admin</a>

app/flatpages/help/copyright.md

@@ -135,7 +135,8 @@ ContentDB editors will check packages to make sure the package page's license ma
 inside the package download, but do not investigate each piece of media or line of code.
 
 If a copyright violation is reported to us, we will unlist the package and contact the author/maintainers.
-Once the problem has been fixed, the package can be restored.
+Once the problem has been fixed, the package can be restored. Repeated copyright infringement may lead to
+permanent bans.
 
 
 ## Where can I get help?

app/flatpages/help/editors.md

@@ -26,7 +26,6 @@ The [Editor Work Queue](/todo/) and related pages contain useful information for
 * The package, release, and screenshot approval queues.
 * Packages which are outdated or are missing tags.
 * A list of forum topics without packages.
-  Editors can create the packages or "discard" them if they don't think it's worth adding them.
 
 ## Editor Notifications
 
@@ -54,5 +53,4 @@ A simplified process for reviewing a package is as follows:
    usually incorrect.
 4. check source, etc links to make sure they work and are correct.
 5. verify that the package has license file that matches what is on the contentdb fields
-6. verify that all assets and code are licensed correctly
-7. if the above steps pass, approve the package, else request changes needed from the author
+6. if the above steps pass, approve the package, else request changes needed from the author

app/flatpages/help/faq.md

@@ -34,7 +34,7 @@ then you can just set a new email in
 [Settings > Email and Notifications](/user/settings/email/).
 
 If you have previously unsubscribed this email, then ContentDB is completely prevented from sending emails to that
-address. You'll need to use a different email address, or [contact rubenwardy](https://rubenwardy.com/contact/) to
+address. You'll need to use a different email address, or [contact the admin]({{ admin_contact_url }}) to
 remove your email from the blacklist.
 
 
@@ -58,6 +58,10 @@ for a guide to making mods and games using Minetest.
 
 See [Installing content](/help/installing/).
 
+### How can my package get more downloads?
+
+See [Creating an appealing ContentDB page](/help/appealing_page/).
+
 
 ## How do I get help?
 

app/flatpages/help/feeds.md

@@ -0,0 +1,16 @@
+title: Feeds
+
+You can follow updates from ContentDB in your RSS feed reader. If in doubt, copy the Atom URL.
+
+* All events: [Atom]({{ url_for('feeds.all_atom') }}) | [JSONFeed]({{ url_for('feeds.all_json') }})
+* New packages: [Atom]({{ url_for('feeds.packages_all_atom') }}) | [JSONFeed]({{ url_for('feeds.packages_all_json') }})
+* New releases: [Atom]({{ url_for('feeds.releases_all_atom') }}) | [JSONFeed]({{ url_for('feeds.releases_all_json') }})
+
+## Package feeds
+
+Follow new releases for a package:
+
+```
+https://content.minetest.net/packages/AUTHOR/NAME/releases_feed.atom
+https://content.minetest.net/packages/AUTHOR/NAME/releases_feed.json
+```

app/flatpages/help/game_support.md

@@ -1,9 +1,5 @@
 title: Supported Games
 
-<p class="alert alert-warning">
-    This feature is experimental
-</p>
-
 ## Why?
 
 The supported/compatible games feature allows mods to specify the games that

app/flatpages/help/metrics.md

@@ -9,11 +9,13 @@ and modern alerting approach".
 Prometheus Metrics can be accessed at [/metrics](/metrics), or you can view them
 on the Grafana instance below.
 
+{% if monitoring_url %}
 <p>
-    <a class="btn btn-primary" href="https://monitor.rubenwardy.com/d/3ELzFy3Wz/contentdb">
+    <a class="btn btn-primary" href="{{ monitoring_url }}">
         View ContentDB on Grafana
     </a>
 </p>
+{% endif %}
 
 ## Metrics
 

app/flatpages/help/non_free.md

@@ -71,3 +71,13 @@ each platforms shows. It doesn't hide anything on Desktop, but hides all mature
 content on Android. You may wish to remove all text from that setting completely, 
 leaving it blank. See [Content Warnings](/help/content_flags/#content-warnings)
 for information on mature content.
+
+## How can I hide non-free packages on the website?
+
+Clicking "Hide non-free packages" in the footer of ContentDB will hide non-free packages from search results.
+It will not hide non-free packages from user profiles.
+
+## See also
+
+* [List of non-free packages](/packages/?flag=nonfree)
+* [Copyright Guide](/help/copyright)

app/flatpages/help/oauth.md

@@ -8,6 +8,11 @@ ContentDB allows you to create an OAuth2 Application and obtain access tokens
 for users.
 
 
+## Scopes
+
+OAuth2 applications can currently only access public user data, using the whoami API.
+
+
 ## Create an OAuth2 Client
 
 Go to Settings > [OAuth2 Applications](/user/apps/) > Create
@@ -64,6 +69,7 @@ If successful, you'll receive:
 
 ```json
 {
+    "success": true,
     "access_token": "access_token",
     "token_type": "Bearer"
 }
@@ -95,12 +101,3 @@ Next, you should check the access token works by getting the user information:
 curl https://content.minetest.net/api/whoami/ \
     -H "Authorization: Bearer YOURTOKEN"
 ```
-
-
-## Scopes
-
-* (no scope) - public data only
-* `user:email`: read user email
-* `package`: write access to packages
-* `package:release`: create and delete releases
-* `package:screenshot`: create, edit, delete screenshots

app/flatpages/help/package_config.md

@@ -70,12 +70,13 @@ It should be a JSON dictionary with one or more of the following optional keys:
 * `license`: A license name, see [/api/licenses/](/api/licenses/).
 * `media_license`: A license name.          
 * `long_description`: Long markdown description.
-* `repo`: Git repo URL.
+* `repo`: Source repository (eg: Git).
 * `website`: Website URL.
 * `issue_tracker`: Issue tracker URL.
 * `forums`: forum topic ID.
 * `video_url`: URL to a video.
 * `donate_url`: URL to a donation page.
+* `translation_url`: URL to send users interested in translating your package.
 
 Use `null` or `[]` to unset fields where relevant.
 

app/flatpages/help/release_webhooks.md

@@ -16,14 +16,21 @@ See [Git Update Detection](/help/update_config/).
 The process is as follows:
 
 1. The user creates an API Token and a webhook to use it.
-2. The user pushes a commit to the git host (Gitlab or Github).
+2. The user pushes a commit to the git host (GitLab or GitHub).
 3. The git host posts a webhook notification to ContentDB, using the API token assigned to it.
 4. ContentDB checks the API token and issues a new release.
+    * If multiple packages match, then only the first will have a release created.
+ 
+### Branch filtering
+
+By default, "New commit" or "push" based webhooks will only work on "master"/"main" branches.
+You can configure the branch used by changing "Branch name" in [Git update detection](update_config).
+
+For example, to support production and beta packages you can have multiple packages with the same VCS repo URL
+but different [Git update detection](update_config) branch names.
+
+Tag-based webhooks are accepted on any branch.
 
-<p class="alert alert-warning">
-    "New commit" or "push" based webhooks will currently only work on branches named `master` or
-    `main`.
-</p>
 
 ## Setting up
 
@@ -36,10 +43,10 @@ The process is as follows:
 5. Set the content type to JSON.
 6. Set the secret to the access token that you copied.
 7. Set the events
-  * If you want a rolling release, choose "just the push event".
-  * Or if you want a stable release cycle based on tags,
-    choose "Let me select" > Branch or tag creation.
+    * If you want a rolling release, choose "just the push event".
+    * Or if you want a stable release cycle based on tags, choose "Let me select" > Branch or tag creation.
 8. Create.
+9. If desired, change [Git update detection](update_config) > Branch name to configure the [branch filtering](#branch-filtering).
 
 ### GitLab
 
@@ -53,6 +60,7 @@ The process is as follows:
     * Or if you want a stable release cycle based on tags,
       choose "Tag push events".
 8. Add webhook.
+9. If desired, change [Git update detection](update_config) > Branch name to configure the [branch filtering](#branch-filtering).
 
 ## Configuring Release Creation
 

app/flatpages/help/top_packages.md

@@ -19,7 +19,7 @@ score       = avg_downloads + reviews_sum;
 ## Pseudo rolling average of downloads
 
 Each package adds 1 to `avg_downloads` for each unique download,
-and then loses 5% (=1/20) of the value every day.
+and then loses 6.66% (=1/15) of the value every day.
 
 This is called a [Frecency](https://en.wikipedia.org/wiki/Frecency) heuristic,
 a measure which combines both frequency and recency.

app/flatpages/policy_and_guidance.md

@@ -37,18 +37,16 @@ See [Content Flags](/help/content_flags/).
 ### 2.2. State of Completion
 
 ContentDB should only currently contain playable content - content which is
-sufficiently complete to be useful to end-users. It's fine to add stuff which
-is still a Work in Progress (WIP) as long as it adds sufficient value;
-MineClone 2 is a good example of a WIP package which may break between releases
-but still has value. Note that this doesn't mean that you should add a thing
-you started working on yesterday, it's worth adding all the basic stuff to
-make your package useful.
+sufficiently complete to be useful to end-users. It's fine to add stuff which is
+still a Work in Progress (WIP) as long as it adds sufficient value; Note that
+this doesn't mean that you should add a thing you started working on yesterday,
+it's worth adding all the basic stuff to make your package useful.
 
-You should make sure to mark Work in Progress stuff as such in the "maintenance status" column,
-as this will help advise players.
+You should make sure to mark Work in Progress stuff as such in the "maintenance
+status" column, as this will help advise players.
 
-Adding non-player facing mods, such as libraries and server tools, is perfectly fine
-and encouraged. ContentDB isn't just for player-facing things, and adding
+Adding non-player facing mods, such as libraries and server tools, is perfectly
+fine and encouraged. ContentDB isn't just for player-facing things, and adding
 libraries allows them to be installed when a mod depends on it.
 
 
@@ -98,8 +96,8 @@ of the content on servers or singleplayer is also not permitted.**
 However, closed sourced licenses are allowed if they allow the above.
 
 If the license you use is not on the list then please select "Other", and we'll
-get around to adding it. We tend to reject custom/untested licenses, and 
-reserve the right to decide whether a license should be included. 
+get around to adding it. We tend to reject custom/untested licenses, and
+reserve the right to decide whether a license should be included.
 
 Please note that the definitions of "free" and "non-free" is the same as that
 of the [Free Software Foundation](https://www.gnu.org/philosophy/free-sw.en.html).
@@ -109,9 +107,8 @@ of the [Free Software Foundation](https://www.gnu.org/philosophy/free-sw.en.html
 It is highly recommended that you use a Free and Open Source software (FOSS)
 license. FOSS licenses result in a sharing community and will increase the
 number of potential users your package has. Using a closed source license will
-result in your package being massively penalised in the search results and
-package lists. See the help page on [non-free licenses](/help/non_free/) for more
-information.
+result in your package not being shown in Minetest by default. See the help page
+on [non-free licenses](/help/non_free/) for more information.
 
 It is recommended that you use a proper license for code with a warranty
 disclaimer, such as the (L)GPL or MIT. You should also use a proper media license
@@ -196,6 +193,8 @@ clearly that it does in the package meta.
 Packages must not ask that users disable mod security (`secure.enable_security`).
 Instead, they should use the insecure environment API.
 
+Packages must not contain obfuscated code.
+
 
 ## 9. Reporting Violations
 

app/flatpages/privacy_policy.md

@@ -1,6 +1,7 @@
 title: Privacy Policy
+---
 
-Last Updated: 2022-01-23
+Last Updated: 2024-04-30
 ([View updates](https://github.com/minetest/contentdb/commits/master/app/flatpages/privacy_policy.md))
 
 ## What Information is Collected
@@ -11,8 +12,9 @@ Last Updated: 2022-01-23
 	* Time
 	* IP address
 	* Page URL
-	* Response status code
+	* Platform and Operating System
 * Preferred language/locale. This defaults to the browser's locale, but can be changed by the user
+* Whether an IP address has downloaded a particular package in the last 14 days
 
 **With an account:**
 
@@ -32,7 +34,7 @@ Please avoid giving other personal information as we do not want it.
 
 ## How this information is used
 
-* Logged HTTP requests may be used for debugging ContentDB.
+* Logged HTTP requests may be used for debugging ContentDB and combating abuse.
 * Email addresses are used to:
     * Provide essential system messages, such as password resets and privacy policy updates.
     * Send notifications - the user may configure this to their needs, including opting out.
@@ -40,6 +42,14 @@ Please avoid giving other personal information as we do not want it.
 * Passwords are used to authenticate the user.
 * The audit log is used to record actions that may be harmful.
 * Preferred language/locale is used to translate emails and the ContentDB interface.
+* Requests (such as downloads) are used for aggregated statistics and for
+  calculating the popularity of packages. For example, download counts are shown
+  for each package and release and there are also download graphs available for
+  each package.
+* Whether an IP address has downloaded a package or release is cached to prevent
+  downloads from being counted multiple times per IP address, but this
+  information is deleted after 14 days.
+* IP addresses are used to monitor and combat abuse.
 * Other information is displayed as part of ContentDB's service.
 
 ## Who has access
@@ -57,44 +67,52 @@ Please avoid giving other personal information as we do not want it.
   They are either public, or visible only to the package author and editors.
 * The complete audit log is visible to moderators.
   Users may see their own audit log actions on their account settings page.
-  Owners, maintainers, and editors may be able to see the actions on a package in the future.
-* Preferred language can only be viewed by this with access to the database or a backup.
+  Owners, maintainers, and editors can see the actions on a package.
+* Preferred language can only be viewed by those with access to the database or a backup.
 * We may be required to share information with law enforcement.
 
+## Third-parties
+
+We do not share any personal information with third parties.
+
+We use <a href="https://sentry.io/">Sentry.io</a> for error logging and performance monitoring.
+
 ## Location
 
 The ContentDB production server is currently located in Germany.
 Backups are stored in the UK.
 Encrypted backups may be stored in other countries, such as the US or EU.
 
-By using this service, you give permission for the data to be moved as needed.
+By using this service, you give permission for the data to be moved within the
+United Kingdom and/or EU.
 
 ## Period of Retention
 
-The server uses log rotation, meaning that any logged HTTP requests will be
-forgotten within a few weeks.
+Logged HTTP requests are automatically deleted within 2 weeks.
 
-Usernames may be kept indefinitely, but other user information will be deleted if
-requested. See below.
+Usernames may be kept indefinitely, but other user information will be deleted
+if requested. See below.
+
+Whether an IP address has downloaded a package or release is deleted after 14 days.
 
 ## Removal Requests
 
-Please [raise a report](/report/?anon=0) if you
-wish to remove your personal information.
+Please [raise a report](/report/?anon=0) if you wish to remove your personal
+information.
 
-ContentDB keeps a record of each username and forum topic on the forums,
-for use in indexing mod/game topics. ContentDB also requires the use of a username
-to uniquely identify a package. Therefore, an author cannot be removed completely
+ContentDB keeps a record of each username and forum topic on the forums, for use
+in indexing mod/game topics. ContentDB also requires the use of a username to
+uniquely identify a package. Therefore, an author cannot be removed completely
 from ContentDB if they have any packages or mod/game topics on the forum.
 
 If we are unable to remove your account for one of the above reasons, your user
 account will instead be wiped and deactivated, ending up exactly like an author
-who has not yet joined ContentDB. All personal information will be removed from the profile,
-and any comments or threads will be deleted.
+who has not yet joined ContentDB. All personal information will be removed from
+the profile, and any comments or threads will be deleted.
 
 ## Future Changes to Privacy Policy
 
-We will alert any future changes to the privacy policy via email and
-via notices on the ContentDB website.
+We will alert any future changes to the privacy policy via notices on the
+ContentDB website.
 
 By continuing to use this service, you agree to the privacy policy.

app/logic/approval_stats.py

@@ -0,0 +1,148 @@
+# ContentDB
+# Copyright (C) 2024  rubenwardy
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program.  If not, see <https://www.gnu.org/licenses/>.
+
+import datetime
+from collections import namedtuple, defaultdict
+from typing import Dict, Optional
+from sqlalchemy import or_
+
+from app.models import AuditLogEntry, db, PackageState
+
+
+class PackageInfo:
+	state: Optional[PackageState]
+	first_submitted: Optional[datetime.datetime]
+	last_change: Optional[datetime.datetime]
+	approved_at: Optional[datetime.datetime]
+	wait_time: int
+	total_approval_time: int
+	is_in_range: bool
+	events: list[tuple[str, str, str]]
+
+	def __init__(self):
+		self.state = None
+		self.first_submitted = None
+		self.last_change = None
+		self.approved_at = None
+		self.wait_time = 0
+		self.total_approval_time = -1
+		self.is_in_range = False
+		self.events = []
+
+	def __lt__(self, other):
+		return self.wait_time < other.wait_time
+
+	def __dict__(self):
+		return {
+			"first_submitted": self.first_submitted.isoformat(),
+			"last_change": self.last_change.isoformat(),
+			"approved_at": self.approved_at.isoformat() if self.approved_at else None,
+			"wait_time": self.wait_time,
+			"total_approval_time": self.total_approval_time if self.total_approval_time >= 0 else None,
+			"events": [ { "date": x[0], "by": x[1], "title": x[2] } for x in self.events ],
+		}
+
+	def add_event(self, created_at: datetime.datetime, causer: str, title: str):
+		self.events.append((created_at.isoformat(), causer, title))
+
+
+def get_state(title: str):
+	if title.startswith("Approved "):
+		return PackageState.APPROVED
+
+	assert title.startswith("Marked ")
+
+	for state in PackageState:
+		if state.value in title:
+			return state
+
+	if "Work in Progress" in title:
+		return PackageState.WIP
+
+	raise Exception(f"Unable to get state for title {title}")
+
+
+Result = namedtuple("Result", "editor_approvals packages_info avg_turnaround_time max_turnaround_time")
+
+
+def _get_approval_statistics(entries: list[AuditLogEntry], start_date: Optional[datetime.datetime] = None, end_date: Optional[datetime.datetime] = None) -> Result:
+	editor_approvals = defaultdict(int)
+	package_info: Dict[str, PackageInfo] = {}
+	ignored_packages = set()
+	turnaround_times: list[int] = []
+
+	for entry in entries:
+		package_id = str(entry.package.get_id())
+		if package_id in ignored_packages:
+			continue
+
+		info = package_info.get(package_id, PackageInfo())
+		package_info[package_id] = info
+
+		is_in_range = (((start_date is None or entry.created_at >= start_date) and
+				(end_date is None or entry.created_at <= end_date)))
+		info.is_in_range = info.is_in_range or is_in_range
+
+		new_state = get_state(entry.title)
+		if new_state == info.state:
+			continue
+
+		info.add_event(entry.created_at, entry.causer.username if entry.causer else None, new_state.value)
+
+		if info.state == PackageState.READY_FOR_REVIEW:
+			seconds = int((entry.created_at - info.last_change).total_seconds())
+			info.wait_time += seconds
+			if is_in_range:
+				turnaround_times.append(seconds)
+
+		if new_state == PackageState.APPROVED:
+			ignored_packages.add(package_id)
+			info.approved_at = entry.created_at
+			if is_in_range:
+				editor_approvals[entry.causer.username] += 1
+			if info.first_submitted is not None:
+				info.total_approval_time = int((entry.created_at - info.first_submitted).total_seconds())
+		elif new_state == PackageState.READY_FOR_REVIEW:
+			if info.first_submitted is None:
+				info.first_submitted = entry.created_at
+
+		info.state = new_state
+		info.last_change = entry.created_at
+
+	packages_info_2 = {}
+	package_count = 0
+	for package_id, info in package_info.items():
+		if info.first_submitted and info.is_in_range:
+			package_count += 1
+			packages_info_2[package_id] = info
+
+	if len(turnaround_times) > 0:
+		avg_turnaround_time = sum(turnaround_times) / len(turnaround_times)
+		max_turnaround_time = max(turnaround_times)
+	else:
+		avg_turnaround_time = 0
+		max_turnaround_time = 0
+
+	return Result(editor_approvals, packages_info_2, avg_turnaround_time, max_turnaround_time)
+
+
+def get_approval_statistics(start_date: Optional[datetime.datetime] = None, end_date: Optional[datetime.datetime] = None) -> Result:
+	entries = AuditLogEntry.query.filter(AuditLogEntry.package).filter(or_(
+		AuditLogEntry.title.like("Approved %"),
+		AuditLogEntry.title.like("Marked %"))
+	).order_by(db.asc(AuditLogEntry.created_at)).all()
+
+	return _get_approval_statistics(entries, start_date, end_date)

app/logic/game_support.py

@@ -1,5 +1,5 @@
 # ContentDB
-# Copyright (C) 2022 rubenwardy
+# Copyright (C) rubenwardy
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU Affero General Public License as published by
@@ -13,32 +13,13 @@
 #
 # You should have received a copy of the GNU Affero General Public License
 # along with this program.  If not, see <https://www.gnu.org/licenses/>.
-
-
 import sys
-from typing import List, Dict
+from typing import List, Dict, Optional, Tuple
 
-import sqlalchemy.orm
+import sqlalchemy
 
-from app.logic.LogicError import LogicError
-from app.models import Package, MetaPackage, PackageType, PackageState, PackageGameSupport
-
-"""
-get_game_support(package):
-	if package is a game:
-		return [ package ]
-
-	for all hard dependencies:
-		support = support AND get_meta_package_support(dep)
-
-	return support
-
-get_meta_package_support(meta):
-	for package implementing mod name:
-		support = support OR get_game_support(package)
-
-	return support
-"""
+from app.models import PackageType, Package, PackageState, PackageGameSupport
+from app.utils import post_bot_message
 
 
 minetest_game_mods = {
@@ -55,123 +36,340 @@ mtg_mod_blacklist = {
 }
 
 
-class GameSupportResolver:
-	session: sqlalchemy.orm.Session
-	checked_packages = set()
-	checked_modnames = set()
-	resolved_packages: Dict[int, set[int]] = {}
-	resolved_modnames: Dict[int, set[int]] = {}
+class GSPackage:
+	author: str
+	name: str
+	type: PackageType
 
-	def __init__(self, session):
-		self.session = session
+	provides: set[str]
+	depends: set[str]
 
-	def resolve_for_meta_package(self, meta: MetaPackage, history: List[str]) -> set[int]:
-		print(f"Resolving for {meta.name}", file=sys.stderr)
+	user_supported_games: set[str]
+	user_unsupported_games: set[str]
+	detected_supported_games: set[str]
+	supports_all_games: bool
 
-		key = meta.name
-		if key in self.resolved_modnames:
-			return self.resolved_modnames.get(key)
+	detection_disabled: bool
 
-		if key in self.checked_modnames:
-			print(f"Error, cycle found: {','.join(history)}", file=sys.stderr)
-			return set()
+	is_confirmed: bool
+	errors: set[str]
 
-		self.checked_modnames.add(key)
+	def __init__(self, author: str, name: str, type: PackageType, provides: set[str]):
+		self.author = author
+		self.name = name
+		self.type = type
+		self.provides = provides
+		self.depends = set()
+		self.user_supported_games = set()
+		self.user_unsupported_games = set()
+		self.detected_supported_games = set()
+		self.supports_all_games = False
+		self.detection_disabled = False
+		self.is_confirmed = type == PackageType.GAME
+		self.errors = set()
 
-		retval = set()
+		# For dodgy games, discard MTG mods
+		if self.type == PackageType.GAME and self.name in mtg_mod_blacklist:
+			self.provides.difference_update(minetest_game_mods)
 
-		for package in meta.packages:
-			if package.state != PackageState.APPROVED:
-				continue
+	@property
+	def id_(self) -> str:
+		return f"{self.author}/{self.name}"
 
-			if meta.name in minetest_game_mods and package.name in mtg_mod_blacklist:
-				continue
+	@property
+	def supported_games(self) -> set[str]:
+		ret = set()
+		ret.update(self.user_supported_games)
+		if not self.detection_disabled:
+			ret.update(self.detected_supported_games)
+		ret.difference_update(self.user_unsupported_games)
+		return ret
 
-			ret = self.resolve(package, history)
-			if len(ret) == 0:
-				retval = set()
+	@property
+	def unsupported_games(self) -> set[str]:
+		return self.user_unsupported_games
+
+	def add_error(self, error: str):
+		print(f"ERROR {self.name}: {error}")
+		return self.errors.add(error)
+
+
+class GameSupport:
+	packages: Dict[str, GSPackage]
+	modified_packages: set[GSPackage]
+
+	def __init__(self):
+		self.packages = {}
+		self.modified_packages = set()
+
+	@property
+	def all_confirmed(self):
+		return all([x.is_confirmed for x in self.packages.values()])
+
+	@property
+	def has_errors(self):
+		return any([len(x.errors) > 0 for x in self.packages.values()])
+
+	@property
+	def error_count(self):
+		return sum([len(x.errors) for x in self.packages.values()])
+
+	@property
+	def all_errors(self) -> set[str]:
+		errors = set()
+		for package in self.packages.values():
+			for err in package.errors:
+				errors.add(package.id_ + ": " + err)
+		return errors
+
+	def add(self, package: GSPackage) -> GSPackage:
+		self.packages[package.id_] = package
+		return package
+
+	def get(self, id_: str) -> Optional[GSPackage]:
+		return self.packages.get(id_)
+
+	def get_all_that_provide(self, modname: str) -> List[GSPackage]:
+		return [package for package in self.packages.values() if modname in package.provides]
+
+	def get_all_that_depend_on(self, modname: str) -> List[GSPackage]:
+		return [package for package in self.packages.values() if modname in package.depends]
+
+	def _get_supported_games_for_modname(self, depend: str, visited: list[str]):
+		print(f"_get_supported_games_for_modname {depend} visited {', '.join(visited)}", file=sys.stderr)
+		dep_supports_all = False
+		for_dep = set()
+		for provider in self.get_all_that_provide(depend):
+			found_in = self._get_supported_games(provider, visited)
+			print(f" - provider for {depend}: {provider.name}: {found_in}", file=sys.stderr)
+			if found_in is None:
+				# Unsupported, keep going
+				pass
+			elif len(found_in) == 0:
+				dep_supports_all = True
 				break
+			else:
+				for_dep.update(found_in)
 
-			retval.update(ret)
+		return dep_supports_all, for_dep
 
-		self.resolved_modnames[key] = retval
-		return retval
+	def _get_supported_games_for_deps(self, package: GSPackage, visited: list[str]) -> Optional[set[str]]:
+		print(f"_get_supported_games_for_deps package {package.name} visited {', '.join(visited)}", file=sys.stderr)
+		ret = set()
 
-	def resolve(self, package: Package, history: List[str]) -> set[int]:
-		key: int = package.id
-		print(f"Resolving for {key}", file=sys.stderr)
+		for depend in package.depends:
+			dep_supports_all, for_dep = self._get_supported_games_for_modname(depend, visited)
 
-		history = history.copy()
-		history.append(package.get_id())
+			if dep_supports_all:
+				# Dep is game independent
+				pass
+			elif len(for_dep) == 0:
+				package.add_error(f"Unable to fulfill dependency {depend}")
+				return None
+			elif len(ret) == 0:
+				ret = for_dep
+			else:
+				ret.intersection_update(for_dep)
+				if len(ret) == 0:
+					package.add_error("Game support conflict, unable to install package on any games")
+					return None
+
+		return ret
+
+	def _get_supported_games(self, package: GSPackage, visited: list[str]) -> Optional[set[str]]:
+		print(f"_get_supported_games package {package.name} visited {', '.join(visited)}", file=sys.stderr)
+		if package.id_ in visited:
+			first_idx = visited.index(package.id_)
+			visited = visited[first_idx:]
+			err = f"Dependency cycle detected: {' -> '.join(visited)} -> {package.id_}"
+			for id_ in visited:
+				package2 = self.get(id_)
+				package2.add_error(err)
+			return None
 
 		if package.type == PackageType.GAME:
-			return {package.id}
+			print(f"_get_supported_games package {package.name} is game", file=sys.stderr)
+			return {package.name}
+		elif package.is_confirmed:
+			print(f"_get_supported_games package {package.name} is confirmed", file=sys.stderr)
+			return package.supported_games
 
-		if key in self.resolved_packages:
-			return self.resolved_packages.get(key)
+		visited = visited.copy()
+		visited.append(package.id_)
 
-		if key in self.checked_packages:
-			print(f"Error, cycle found: {','.join(history)}", file=sys.stderr)
-			return set()
+		ret = self._get_supported_games_for_deps(package, visited)
+		if ret is None:
+			assert len(package.errors) > 0
+			return None
 
-		self.checked_packages.add(key)
+		ret = ret.copy()
+		ret.difference_update(package.user_unsupported_games)
+		package.detected_supported_games = ret
+		self.modified_packages.add(package)
 
-		if package.type != PackageType.MOD:
-			raise LogicError(500, "Got non-mod")
+		if len(ret) > 0:
+			for supported in package.user_supported_games:
+				if supported not in ret:
+					package.add_error(f"`{supported}` is specified in supported_games but it is impossible to run {package.name} in that game. " +
+							f"Its dependencies can only be fulfilled in {', '.join([f'`{x}`' for x in ret])}. " +
+							"Check your hard dependencies.")
 
-		retval = set()
+			if package.supports_all_games:
+				package.add_error(
+						"This package cannot support all games as some dependencies require specific game(s): " +
+						", ".join([f'`{x}`' for x in ret]))
 
-		for dep in package.dependencies.filter_by(optional=False).all():
-			ret = self.resolve_for_meta_package(dep.meta_package, history)
-			if len(ret) == 0:
-				continue
-			elif len(retval) == 0:
-				retval.update(ret)
-			else:
-				retval.intersection_update(ret)
-				if len(retval) == 0:
-					raise LogicError(500, f"Detected game support contradiction, {key} may not be compatible with any games")
+		package.is_confirmed = True
+		return package.supported_games
 
-		self.resolved_packages[key] = retval
-		return retval
+	def on_update(self, package: GSPackage, old_provides: Optional[set[str]] = None):
+		to_update = {package}
+		checked = set()
 
-	def init_all(self) -> None:
-		for package in self.session.query(Package).filter(Package.type == PackageType.MOD, Package.state != PackageState.DELETED).all():
-			retval = self.resolve(package, [])
-			for game_id in retval:
-				game = self.session.query(Package).get(game_id)
-				support = PackageGameSupport(package, game, 1, True)
-				self.session.add(support)
+		while len(to_update) > 0:
+			current_package = to_update.pop()
+			print(f"on_update package {current_package.name}", file=sys.stderr)
+			if current_package.id_ in self.packages and current_package.type != PackageType.GAME:
+				self._get_supported_games(current_package, [])
 
-	"""
-	Update game supported package on a package, given the confidence.
-	
-	Higher confidences outweigh lower ones.
-	"""
-	def set_supported(self, package: Package, game_is_supported: Dict[int, bool], confidence: int):
-		previous_supported: Dict[int, PackageGameSupport] = {}
-		for support in package.supported_games.all():
-			previous_supported[support.game.id] = support
+			provides = current_package.provides
+			if current_package == package and old_provides is not None:
+				provides = provides.union(old_provides)
 
-		for game_id, supports in game_is_supported.items():
-			game = self.session.query(Package).get(game_id)
-			lookup = previous_supported.pop(game_id, None)
-			if lookup is None:
-				support = PackageGameSupport(package, game, confidence, supports)
-				self.session.add(support)
-			elif lookup.confidence <= confidence:
-				lookup.supports = supports
-				lookup.confidence = confidence
+			for modname in provides:
+				for depending_package in self.get_all_that_depend_on(modname):
+					if depending_package not in checked:
+						if depending_package.id_ in self.packages and depending_package.type != PackageType.GAME:
+							depending_package.is_confirmed = False
+							depending_package.detected_supported_games = []
 
-		for game, support in previous_supported.items():
-			if support.confidence == confidence:
-				self.session.delete(support)
+						to_update.add(depending_package)
+						checked.add(depending_package)
 
-	def update(self, package: Package) -> None:
-		game_is_supported = {}
-		if package.enable_game_support_detection:
-			retval = self.resolve(package, [])
-			for game_id in retval:
-				game_is_supported[game_id] = True
+	def on_remove(self, package: GSPackage):
+		del self.packages[package.id_]
+		self.on_update(package)
 
-		self.set_supported(package, game_is_supported, 1)
+	def on_first_run(self):
+		for package in self.packages.values():
+			if not package.is_confirmed:
+				self.on_update(package)
+
+
+def _convert_package(support: GameSupport, package: Package) -> GSPackage:
+	# Unapproved packages shouldn't be considered to fulfill anything
+	provides = set()
+	if package.state == PackageState.APPROVED:
+		provides = set([x.name for x in package.provides])
+
+	gs_package = GSPackage(package.author.username, package.name, package.type, provides)
+	gs_package.depends = set([x.meta_package.name for x in package.dependencies if not x.optional])
+	gs_package.detection_disabled = not package.enable_game_support_detection
+	gs_package.supports_all_games = package.supports_all_games
+
+	existing_game_support = (package.supported_games
+			.filter(PackageGameSupport.game.has(state=PackageState.APPROVED),
+					PackageGameSupport.confidence > 5)
+			.all())
+	if not package.supports_all_games:
+		gs_package.user_supported_games = [x.game.name for x in existing_game_support if x.supports]
+	gs_package.user_unsupported_games = [x.game.name for x in existing_game_support if not x.supports]
+	return support.add(gs_package)
+
+
+def _create_instance(session: sqlalchemy.orm.Session) -> GameSupport:
+	support = GameSupport()
+
+	packages: List[Package] = (session.query(Package)
+			.filter(Package.state == PackageState.APPROVED, Package.type.in_([PackageType.GAME, PackageType.MOD]))
+			.all())
+
+	for package in packages:
+		_convert_package(support, package)
+
+	return support
+
+
+def _persist(session: sqlalchemy.orm.Session, support: GameSupport):
+	for gs_package in support.packages.values():
+		if len(gs_package.errors) != 0:
+			msg = "\n".join([f"- {x}" for x in gs_package.errors])
+			package = session.query(Package).filter(
+					Package.author.has(username=gs_package.author),
+					Package.name == gs_package.name).one()
+			post_bot_message(package, "Error when checking game support", msg, session)
+
+	for gs_package in support.modified_packages:
+		if not gs_package.detection_disabled:
+			package = session.query(Package).filter(
+					Package.author.has(username=gs_package.author),
+					Package.name == gs_package.name).one()
+
+			# Clear existing
+			session.query(PackageGameSupport) \
+				.filter_by(package=package, confidence=1) \
+				.delete()
+
+			# Add new
+			supported_games = gs_package.supported_games \
+				.difference(gs_package.user_supported_games)
+			for game_name in supported_games:
+				game_id = session.query(Package.id) \
+					.filter(Package.type == PackageType.GAME, Package.name == game_name, Package.state == PackageState.APPROVED) \
+					.one()[0]
+
+				new_support = PackageGameSupport()
+				new_support.package = package
+				new_support.game_id = game_id
+				new_support.confidence = 1
+				new_support.supports = True
+				session.add(new_support)
+
+
+def game_support_update(session: sqlalchemy.orm.Session, package: Package, old_provides: Optional[set[str]]) -> set[str]:
+	support = _create_instance(session)
+	gs_package = support.get(package.get_id())
+	if gs_package is None:
+		gs_package = _convert_package(support, package)
+	support.on_update(gs_package, old_provides)
+	_persist(session, support)
+	return gs_package.errors
+
+
+def game_support_update_all(session: sqlalchemy.orm.Session):
+	support = _create_instance(session)
+	support.on_first_run()
+	_persist(session, support)
+
+
+def game_support_remove(session: sqlalchemy.orm.Session, package: Package):
+	support = _create_instance(session)
+	gs_package = support.get(package.get_id())
+	if gs_package is None:
+		gs_package = _convert_package(support, package)
+	support.on_remove(gs_package)
+	_persist(session, support)
+
+
+def game_support_set(session, package: Package, game_is_supported: Dict[int, bool], confidence: int):
+	previous_supported: Dict[int, PackageGameSupport] = {}
+	for support in package.supported_games.all():
+		previous_supported[support.game.id] = support
+
+	for game_id, supports in game_is_supported.items():
+		game = session.query(Package).get(game_id)
+		lookup = previous_supported.pop(game_id, None)
+		if lookup is None:
+			support = PackageGameSupport()
+			support.package = package
+			support.game = game
+			support.confidence = confidence
+			support.supports = supports
+			session.add(support)
+		elif lookup.confidence <= confidence:
+			lookup.supports = supports
+			lookup.confidence = confidence
+
+	for game, support in previous_supported.items():
+		if support.confidence == confidence:
+			session.delete(support)

app/logic/package_approval.py

@@ -0,0 +1,202 @@
+# ContentDB
+# Copyright (C) rubenwardy
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program.  If not, see <https://www.gnu.org/licenses/>.
+
+from typing import List, Tuple, Union, Optional
+
+from flask_babel import lazy_gettext, LazyString
+from sqlalchemy import and_, or_
+
+from app.models import Package, PackageType, PackageState, PackageRelease, db, MetaPackage, ForumTopic, User, \
+	Permission, UserRank
+
+
+class PackageValidationNote:
+	# level is danger, warning, or info
+	level: str
+	message: LazyString
+	buttons: List[Tuple[str, LazyString]]
+
+	# False to prevent "Approve"
+	allow_approval: bool
+
+	# False to prevent "Submit for Approval"
+	allow_submit: bool
+
+	def __init__(self, level: str, message: LazyString, allow_approval: bool, allow_submit: bool):
+		self.level = level
+		self.message = message
+		self.buttons = []
+		self.allow_approval = allow_approval
+		self.allow_submit = allow_submit
+
+	def add_button(self, url: str, label: LazyString) -> "PackageValidationNote":
+		self.buttons.append((url, label))
+		return self
+
+
+def is_package_name_taken(normalised_name: str) -> bool:
+	return Package.query.filter(
+		and_(Package.state == PackageState.APPROVED,
+			 or_(Package.name == normalised_name,
+				 Package.name == normalised_name + "_game"))).count() > 0
+
+
+def get_conflicting_mod_names(package: Package) -> set[str]:
+	conflicting_modnames = (db.session.query(MetaPackage.name)
+			.filter(MetaPackage.id.in_([mp.id for mp in package.provides]))
+			.filter(MetaPackage.packages.any(and_(Package.id != package.id, Package.state == PackageState.APPROVED)))
+			.all())
+	conflicting_modnames += (db.session.query(ForumTopic.name)
+			.filter(ForumTopic.name.in_([mp.name for mp in package.provides]))
+			.filter(ForumTopic.topic_id != package.forums)
+			.filter(~ db.exists().where(Package.forums == ForumTopic.topic_id))
+			.order_by(db.asc(ForumTopic.name), db.asc(ForumTopic.title))
+			.all())
+	return set([x[0] for x in conflicting_modnames])
+
+
+def count_packages_with_forum_topic(topic_id: int) -> int:
+	return Package.query.filter(Package.forums == topic_id, Package.state != PackageState.DELETED).count() > 1
+
+
+def get_forum_topic(topic_id: int) -> Optional[ForumTopic]:
+	return ForumTopic.query.get(topic_id)
+
+
+def validate_package_for_approval(package: Package) -> List[PackageValidationNote]:
+	retval: List[PackageValidationNote] = []
+
+	def template(level: str, allow_approval: bool, allow_submit: bool):
+		def inner(msg: LazyString):
+			note = PackageValidationNote(level, msg, allow_approval, allow_submit)
+			retval.append(note)
+			return note
+
+		return inner
+
+	danger = template("danger", allow_approval=False, allow_submit=False)
+	warning = template("warning", allow_approval=True, allow_submit=True)
+	info = template("info", allow_approval=False, allow_submit=True)
+
+	if package.type != PackageType.MOD and is_package_name_taken(package.normalised_name):
+		danger(lazy_gettext("A package already exists with this name. Please see Policy and Guidance 3"))
+
+	if package.releases.filter(PackageRelease.task_id.is_(None)).count() == 0:
+		if package.releases.count() == 0:
+			message = lazy_gettext("You need to create a release before this package can be approved.")
+		else:
+			message = lazy_gettext("Release is still importing, or has an error.")
+
+		danger(message) \
+			.add_button(package.get_url("packages.create_release"), lazy_gettext("Create release")) \
+			.add_button(package.get_url("packages.setup_releases"), lazy_gettext("Set up releases"))
+
+		# Don't bother validating any more until we have a release
+		return retval
+
+	if (package.type == PackageType.GAME or package.type == PackageType.TXP) and \
+			package.screenshots.count() == 0:
+		danger(lazy_gettext("You need to add at least one screenshot."))
+
+	missing_deps = package.get_missing_hard_dependencies_query().all()
+	if len(missing_deps) > 0:
+		missing_deps = ", ".join([ x.name for x in missing_deps])
+		danger(lazy_gettext(
+			"The following hard dependencies need to be added to ContentDB first: %(deps)s", deps=missing_deps))
+
+	if package.type != PackageType.GAME and not package.supports_all_games and package.supported_games.count() == 0:
+		danger(lazy_gettext(
+			"What games does your package support? Please specify on the supported games page", deps=missing_deps)) \
+			.add_button(package.get_url("packages.game_support"), lazy_gettext("Supported Games"))
+
+	if "Other" in package.license.name or "Other" in package.media_license.name:
+		info(lazy_gettext("Please wait for the license to be added to CDB."))
+
+	# Check similar mod name
+	conflicting_modnames = set()
+	if package.type != PackageType.TXP:
+		conflicting_modnames = get_conflicting_mod_names(package)
+
+	if len(conflicting_modnames) > 4:
+		warning(lazy_gettext("Please make sure that this package has the right to the names it uses."))
+	elif len(conflicting_modnames) > 0:
+		names_list = list(conflicting_modnames)
+		names_list.sort()
+		warning(lazy_gettext("Please make sure that this package has the right to the names %(names)s",
+				names=", ".join(names_list))) \
+			.add_button(package.get_url('packages.similar'), lazy_gettext("See more"))
+
+	# Check forum topic
+	if package.state != PackageState.APPROVED and package.forums is not None:
+		if count_packages_with_forum_topic(package.forums) > 1:
+			danger("<b>" + lazy_gettext("Error: Another package already uses this forum topic!") + "</b>")
+
+		topic = get_forum_topic(package.forums)
+		if topic is not None:
+			if topic.author != package.author:
+				danger("<b>" + lazy_gettext("Error: Forum topic author doesn't match package author.") + "</b>")
+		elif package.type != PackageType.TXP:
+			warning(lazy_gettext("Warning: Forum topic not found. The topic may have been created since the last forum crawl."))
+
+	return retval
+
+
+PACKAGE_STATE_FLOW = {
+	PackageState.WIP: {PackageState.READY_FOR_REVIEW},
+	PackageState.CHANGES_NEEDED: {PackageState.READY_FOR_REVIEW},
+	PackageState.READY_FOR_REVIEW: {PackageState.WIP, PackageState.CHANGES_NEEDED, PackageState.APPROVED},
+	PackageState.APPROVED: {PackageState.CHANGES_NEEDED},
+	PackageState.DELETED: {PackageState.READY_FOR_REVIEW},
+}
+
+
+def can_move_to_state(package: Package, user: User, new_state: Union[str, PackageState]) -> bool:
+	if not user.is_authenticated:
+		return False
+
+	if type(new_state) == str:
+		new_state = PackageState[new_state]
+	elif type(new_state) != PackageState:
+		raise Exception("Unknown state given to can_move_to_state()")
+
+	if new_state not in PACKAGE_STATE_FLOW[package.state]:
+		return False
+
+	if new_state == PackageState.READY_FOR_REVIEW or new_state == PackageState.APPROVED:
+		# Can the user approve?
+		if new_state == PackageState.APPROVED and not package.check_perm(user, Permission.APPROVE_NEW):
+			return False
+
+		# Must be able to edit or approve package to change its state
+		if not (package.check_perm(user, Permission.APPROVE_NEW) or package.check_perm(user, Permission.EDIT_PACKAGE)):
+			return False
+
+		# Are there any validation warnings?
+		validation_notes = validate_package_for_approval(package)
+		for note in validation_notes:
+			if not note.allow_submit or (new_state == PackageState.APPROVED and not note.allow_approval):
+				return False
+
+		return True
+
+	elif new_state == PackageState.CHANGES_NEEDED:
+		return package.check_perm(user, Permission.APPROVE_NEW)
+
+	elif new_state == PackageState.WIP:
+		return package.check_perm(user, Permission.EDIT_PACKAGE) and \
+			(user in package.maintainers or user.rank.at_least(UserRank.ADMIN))
+
+	return True

app/logic/package_validator.py

@@ -1,56 +0,0 @@
-# ContentDB
-# Copyright (C) rubenwardy
-#
-# This program is free software: you can redistribute it and/or modify
-# it under the terms of the GNU Affero General Public License as published by
-# the Free Software Foundation, either version 3 of the License, or
-# (at your option) any later version.
-#
-# This program is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-# GNU Affero General Public License for more details.
-#
-# You should have received a copy of the GNU Affero General Public License
-# along with this program.  If not, see <https://www.gnu.org/licenses/>.
-
-from collections import namedtuple
-from typing import List
-
-from flask_babel import lazy_gettext
-from sqlalchemy import and_, or_
-
-from app.models import Package, PackageType, PackageState, PackageRelease
-
-
-ValidationError = namedtuple("ValidationError", "status message")
-
-
-def validate_package_for_approval(package: Package) -> List[ValidationError]:
-	retval: List[ValidationError] = []
-
-	normalised_name = package.getNormalisedName()
-	if package.type != PackageType.MOD and Package.query.filter(
-			and_(Package.state == PackageState.APPROVED,
-				or_(Package.name == normalised_name,
-					Package.name == normalised_name + "_game"))).count() > 0:
-		retval.append(("danger", lazy_gettext("A package already exists with this name. Please see Policy and Guidance 3")))
-
-	if package.releases.filter(PackageRelease.task_id == None).count() == 0:
-		retval.append(("danger", lazy_gettext("A release is required before this package can be approved.")))
-		# Don't bother validating any more until we have a release
-		return retval
-
-	missing_deps = package.get_missing_hard_dependencies_query().all()
-	if len(missing_deps) > 0:
-		retval.append(("danger", lazy_gettext(
-			"The following hard dependencies need to be added to ContentDB first: %(deps)s", deps=missing_deps)))
-
-	if (package.type == package.type.GAME or package.type == package.type.TXP) and \
-		package.screenshots.count() == 0:
-		retval.append(("danger", lazy_gettext("You need to add at least one screenshot.")))
-
-	if "Other" in package.license.name or "Other" in package.media_license.name:
-		retval.append(("info", lazy_gettext("Please wait for the license to be added to CDB.")))
-
-	return retval

app/logic/packages.py

@@ -23,8 +23,8 @@ from flask_babel import lazy_gettext, LazyString
 
 from app.logic.LogicError import LogicError
 from app.models import User, Package, PackageType, MetaPackage, Tag, ContentWarning, db, Permission, AuditSeverity, \
-	License, UserRank, PackageDevState
-from app.utils import add_audit_log, has_blocked_domains, diff_dictionaries, describe_difference
+	License, PackageDevState, PackageState
+from app.utils import add_audit_log, has_blocked_domains, diff_dictionaries, describe_difference, normalize_line_endings
 from app.utils.url import clean_youtube_url
 
 
@@ -66,6 +66,7 @@ ALLOWED_FIELDS = {
 	"forums": int,
 	"video_url": str,
 	"donate_url": str,
+	"translation_url": str,
 }
 
 ALIASES = {
@@ -102,8 +103,7 @@ def validate(data: dict):
 		if value is not None:
 			check(value.startswith("http://") or value.startswith("https://"),
 					key + " must start with http:// or https://")
-
-			check(validators.url(value, public=True), key + " must be a valid URL")
+			check(validators.url(value), key + " must be a valid URL")
 
 
 def do_edit_package(user: User, package: Package, was_new: bool, was_web: bool, data: dict,
@@ -125,13 +125,19 @@ def do_edit_package(user: User, package: Package, was_new: bool, was_web: bool,
 
 	validate(data)
 
-	for field in ["short_desc", "desc", "website", "issueTracker", "repo", "video_url", "donate_url"]:
+	for field in ["short_desc", "desc", "website", "issueTracker", "repo", "video_url", "donate_url", "translation_url"]:
 		if field in data and has_blocked_domains(data[field], user.username,
 					f"{field} of {package.get_id()}"):
 			raise LogicError(403, lazy_gettext("Linking to blocked sites is not allowed"))
 
 	if "type" in data:
-		data["type"] = PackageType.coerce(data["type"])
+		new_type = PackageType.coerce(data["type"])
+		if new_type == package.type:
+			pass
+		elif package.state != PackageState.APPROVED:
+			package.type = new_type
+		else:
+			raise LogicError(403, lazy_gettext("You cannot change package type once approved"))
 
 	if "dev_state" in data:
 		data["dev_state"] = PackageDevState.coerce(data["dev_state"])
@@ -142,13 +148,16 @@ def do_edit_package(user: User, package: Package, was_new: bool, was_web: bool,
 	if "media_license" in data:
 		data["media_license"] = get_license(data["media_license"])
 
+	if "desc" in data:
+		data["desc"] = normalize_line_endings(data["desc"])
+
 	if "video_url" in data and data["video_url"] is not None:
 		data["video_url"] = clean_youtube_url(data["video_url"]) or data["video_url"]
 		if "dQw4w9WgXcQ" in data["video_url"]:
 			raise LogicError(403, "Never gonna give you up / Never gonna let you down / Never gonna run around and desert you")
 
-	for key in ["name", "title", "short_desc", "desc", "type", "dev_state", "license", "media_license",
-			"repo", "website", "issueTracker", "forums", "video_url", "donate_url"]:
+	for key in ["name", "title", "short_desc", "desc", "dev_state", "license", "media_license",
+			"repo", "website", "issueTracker", "forums", "video_url", "donate_url", "translation_url"]:
 		if key in data:
 			setattr(package, key, data[key])
 

app/logic/releases.py

@@ -16,6 +16,7 @@
 
 import datetime
 import re
+from typing import Optional
 
 from celery import uuid
 from flask_babel import lazy_gettext
@@ -24,7 +25,7 @@ from app.logic.LogicError import LogicError
 from app.logic.uploads import upload_file
 from app.models import PackageRelease, db, Permission, User, Package, MinetestRelease
 from app.tasks.importtasks import make_vcs_release, check_zip_release
-from app.utils import AuditSeverity, add_audit_log, nonempty_or_none
+from app.utils import AuditSeverity, add_audit_log, nonempty_or_none, normalize_line_endings
 
 
 def check_can_create_release(user: User, package: Package):
@@ -32,18 +33,20 @@ def check_can_create_release(user: User, package: Package):
 		raise LogicError(403, lazy_gettext("You don't have permission to make releases"))
 
 	five_minutes_ago = datetime.datetime.now() - datetime.timedelta(minutes=5)
-	count = package.releases.filter(PackageRelease.releaseDate > five_minutes_ago).count()
+	count = package.releases.filter(PackageRelease.created_at > five_minutes_ago).count()
 	if count >= 5:
 		raise LogicError(429, lazy_gettext("You've created too many releases for this package in the last 5 minutes, please wait before trying again"))
 
 
-def do_create_vcs_release(user: User, package: Package, title: str, ref: str,
+def do_create_vcs_release(user: User, package: Package, name: str, title: Optional[str], release_notes: Optional[str], ref: str,
 		min_v: MinetestRelease = None, max_v: MinetestRelease = None, reason: str = None):
 	check_can_create_release(user, package)
 
 	rel = PackageRelease()
 	rel.package = package
-	rel.title   = title
+	rel.name    = name
+	rel.title   = title or name
+	rel.release_notes = normalize_line_endings(release_notes)
 	rel.url     = ""
 	rel.task_id = uuid()
 	rel.min_rel = min_v
@@ -63,7 +66,7 @@ def do_create_vcs_release(user: User, package: Package, title: str, ref: str,
 	return rel
 
 
-def do_create_zip_release(user: User, package: Package, title: str, file,
+def do_create_zip_release(user: User, package: Package, name: str, title: Optional[str], release_notes: Optional[str], file,
 		min_v: MinetestRelease = None, max_v: MinetestRelease = None, reason: str = None,
 		commit_hash: str = None):
 	check_can_create_release(user, package)
@@ -77,7 +80,9 @@ def do_create_zip_release(user: User, package: Package, title: str, file,
 
 	rel = PackageRelease()
 	rel.package = package
-	rel.title   = title
+	rel.name    = name
+	rel.title   = title or name
+	rel.release_notes = normalize_line_endings(release_notes)
 	rel.url     = uploaded_url
 	rel.task_id = uuid()
 	rel.commit_hash = commit_hash

app/logic/scope.py

@@ -1,6 +0,0 @@
-from app.models import APIToken
-
-
-class Scope:
-	def copy_to_token(self, token: APIToken):
-		pass

app/logic/screenshots.py

@@ -31,7 +31,7 @@ def do_create_screenshot(user: User, package: Package, title: str, file, is_cove
 	if count >= 20:
 		raise LogicError(429, lazy_gettext("Too many requests, please wait before trying again"))
 
-	uploaded_url, uploaded_path = upload_file(file, "image", lazy_gettext("a PNG or JPG image file"))
+	uploaded_url, uploaded_path = upload_file(file, "image", lazy_gettext("a PNG, JPEG, or WebP image file"))
 
 	counter = 1
 	for screenshot in package.screenshots.all():

app/logic/uploads.py

@@ -28,7 +28,7 @@ def get_extension(filename):
 	return filename.rsplit(".", 1)[1].lower() if "." in filename else None
 
 
-ALLOWED_IMAGES = {"jpeg", "png"}
+ALLOWED_IMAGES = {"jpeg", "png", "webp"}
 
 
 def is_allowed_image(data):
@@ -43,7 +43,7 @@ def upload_file(file, file_type, file_type_desc):
 
 	is_image = False
 	if file_type == "image":
-		allowed_extensions = ["jpg", "jpeg", "png"]
+		allowed_extensions = ["jpg", "png", "webp"]
 		is_image = True
 	elif file_type == "zip":
 		allowed_extensions = ["zip"]
@@ -51,6 +51,9 @@ def upload_file(file, file_type, file_type_desc):
 		raise Exception("Invalid fileType")
 
 	ext = get_extension(file.filename)
+	if ext == "jpeg":
+		ext = "jpg"
+
 	if ext is None or ext not in allowed_extensions:
 		raise LogicError(400, lazy_gettext("Please upload %(file_desc)s", file_desc=file_type_desc))
 

app/logic/users.py

@@ -0,0 +1,60 @@
+from typing import Optional
+
+from flask import flash, redirect, url_for
+from flask_babel import gettext, get_locale
+from sqlalchemy import or_
+from werkzeug import Response
+
+from app.models import User, UserRank, PackageAlias, EmailSubscription, UserNotificationPreferences, db
+from app.utils import is_username_valid
+from app.tasks.emails import send_anon_email
+
+
+def create_user(username: str, display_name: str, email: Optional[str], oauth_provider: Optional[str] = None) -> None | Response | User:
+	if not is_username_valid(username):
+		flash(gettext("Username is invalid"))
+		return
+
+	user_by_name = User.query.filter(or_(
+			User.username == username,
+			User.username == display_name,
+			User.display_name == display_name,
+			User.forums_username == username,
+			User.github_username == username)).first()
+	if user_by_name:
+		if user_by_name.rank == UserRank.NOT_JOINED and user_by_name.forums_username:
+			flash(gettext("An account already exists for that username but hasn't been claimed yet."), "danger")
+			return redirect(url_for("users.claim_forums", username=user_by_name.forums_username))
+		elif oauth_provider:
+			flash(gettext("Unable to create an account as the username is already taken. "
+					"If you meant to log in, you need to connect %(provider)s to your account first", provider=oauth_provider), "danger")
+			return
+		else:
+			flash(gettext("That username/display name is already in use, please choose another."), "danger")
+			return
+
+	alias_by_name = (PackageAlias.query
+			.filter(or_(PackageAlias.author == username, PackageAlias.author == display_name))
+			.first())
+	if alias_by_name:
+		flash(gettext("Unable to create an account as the username was used in the past."), "danger")
+		return
+
+	if email:
+		user_by_email = User.query.filter_by(email=email).first()
+		if user_by_email:
+			send_anon_email.delay(email, get_locale().language, gettext("Email already in use"),
+				gettext("We were unable to create the account as the email is already in use by %(display_name)s. Try a different email address.",
+						display_name=user_by_email.display_name))
+			return redirect(url_for("users.email_sent"))
+		elif EmailSubscription.query.filter_by(email=email, blacklisted=True).count() > 0:
+			flash(gettext("That email address has been unsubscribed/blacklisted, and cannot be used"), "danger")
+			return
+
+	user = User(username, False, email)
+	user.notification_preferences = UserNotificationPreferences(user)
+	if display_name:
+		user.display_name = display_name
+	db.session.add(user)
+
+	return user

app/maillogger.py

@@ -1,115 +0,0 @@
-# ContentDB
-# Copyright (C) rubenwardy
-#
-# This program is free software: you can redistribute it and/or modify
-# it under the terms of the GNU Affero General Public License as published by
-# the Free Software Foundation, either version 3 of the License, or
-# (at your option) any later version.
-#
-# This program is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-# GNU Affero General Public License for more details.
-#
-# You should have received a copy of the GNU Affero General Public License
-# along with this program.  If not, see <https://www.gnu.org/licenses/>.
-
-import logging
-
-from app.tasks.emails import send_user_email
-
-
-def _has_newline(line):
-	"""Used by has_bad_header to check for \\r or \\n"""
-	if line and ("\r" in line or "\n" in line):
-		return True
-	return False
-
-
-def _is_bad_subject(subject):
-	"""Copied from: flask_mail.py class Message def has_bad_headers"""
-	if _has_newline(subject):
-		for linenum, line in enumerate(subject.split("\r\n")):
-			if not line:
-				return True
-			if linenum > 0 and line[0] not in "\t ":
-				return True
-			if _has_newline(line):
-				return True
-			if len(line.strip()) == 0:
-				return True
-	return False
-
-
-class FlaskMailSubjectFormatter(logging.Formatter):
-	def format(self, record):
-		record.message = record.getMessage()
-		if self.usesTime():
-			record.asctime = self.formatTime(record, self.datefmt)
-		s = self.formatMessage(record)
-		return s
-
-
-class FlaskMailTextFormatter(logging.Formatter):
-	pass
-
-
-class FlaskMailHTMLFormatter(logging.Formatter):
-	def formatException(self, exc_info):
-		formatted_exception = logging.Handler.formatException(self, exc_info)
-		return "<pre>%s</pre>" % formatted_exception
-	def formatStack(self, stack_info):
-		return "<pre>%s</pre>" % stack_info
-
-
-# see: https://github.com/python/cpython/blob/3.6/Lib/logging/__init__.py (class Handler)
-
-class FlaskMailHandler(logging.Handler):
-	def __init__(self, send_to, subject_template, level=logging.NOTSET):
-		logging.Handler.__init__(self, level)
-		self.send_to = send_to
-		self.subject_template = subject_template
-
-	def setFormatter(self, text_fmt):
-		"""
-		Set the formatters for this handler. Provide at least one formatter.
-		When no text_fmt is provided, no text-part is created for the email body.
-		"""
-		assert text_fmt != None, "At least one formatter should be provided"
-		if type(text_fmt)==str:
-			text_fmt = FlaskMailTextFormatter(text_fmt)
-		self.formatter = text_fmt
-
-	def getSubject(self, record):
-		fmt = FlaskMailSubjectFormatter(self.subject_template)
-		subject = fmt.format(record)
-		# Since templates can cause header problems, and we rather have an incomplete email then an error, we fix this
-		if _is_bad_subject(subject):
-			subject="FlaskMailHandler log-entry from ContentDB [original subject is replaced, because it would result in a bad header]"
-		return subject
-
-	def emit(self, record):
-		subject = self.getSubject(record)
-		text = self.format(record)				if self.formatter	  else None
-		html = "<pre>{}</pre>".format(text)
-
-		if "The recipient has exceeded message rate limit. Try again later" in subject:
-			return
-
-		for email in self.send_to:
-			send_user_email.delay(email, "en", subject, text, html)
-
-
-def build_handler(app):
-	subject_template = "ContentDB %(message)s (%(module)s > %(funcName)s)"
-	text_template = ("Message type: %(levelname)s\n"
-		"Location: %(pathname)s:%(lineno)d\n"
-		"Module:   %(module)s\n"
-		"Function: %(funcName)s\n"
-		"Time:     %(asctime)s\n"
-		"Message: %(message)s\n\n")
-
-	mail_handler = FlaskMailHandler(app.config["MAIL_UTILS_ERROR_SEND_TO"], subject_template)
-	mail_handler.setLevel(logging.ERROR)
-	mail_handler.setFormatter(text_template)
-	return mail_handler

app/markdown.py

@@ -15,6 +15,7 @@
 # along with this program.  If not, see <https://www.gnu.org/licenses/>.
 
 from functools import partial
+from urllib.parse import urljoin
 
 import bleach
 from bleach import Cleaner
@@ -48,6 +49,8 @@ ALLOWED_TAGS = {
 	"img",
 	"table", "thead", "tbody", "tr", "th", "td",
 	"div", "span", "del", "s",
+	"details",
+	"summary",
 }
 
 ALLOWED_CSS = [
@@ -74,6 +77,7 @@ ALLOWED_ATTRIBUTES = {
 	"code": allow_class,
 	"div": allow_class,
 	"span": allow_class,
+	"table": ["id"],
 }
 
 ALLOWED_PROTOCOLS = {"http", "https", "mailto"}
@@ -202,3 +206,9 @@ def get_user_mentions(html: str) -> set:
 	soup = BeautifulSoup(html, "html.parser")
 	links = soup.select("a[data-username]")
 	return set([x.get("data-username") for x in links])
+
+
+def get_links(html: str, url: str) -> set:
+	soup = BeautifulSoup(html, "html.parser")
+	links = soup.select("a[href]")
+	return set([urljoin(url, x.get("href")) for x in links])

app/models/__init__.py

@@ -52,38 +52,8 @@ class APIToken(db.Model):
 	client    = db.relationship("OAuthClient", foreign_keys=[client_id], back_populates="tokens")
 	auth_code = db.Column(db.String(34), unique=True, nullable=True)
 
-	scope_user_email = db.Column(db.Boolean, nullable=False, default=False)
-	scope_package = db.Column(db.Boolean, nullable=False, default=False)
-	scope_package_release = db.Column(db.Boolean, nullable=False, default=False)
-	scope_package_screenshot = db.Column(db.Boolean, nullable=False, default=False)
-
-	def get_scopes(self) -> set[str]:
-		ret = set()
-		if self.scope_user_email:
-			ret.add("user:email")
-		if self.scope_package:
-			ret.add("package")
-		if self.scope_package_release:
-			ret.add("package:release")
-		if self.scope_package_screenshot:
-			ret.add("package:screenshot")
-		return ret
-
-	def set_scopes(self, v: set[str]):
-		def pop(key: str):
-			if key in v:
-				v.remove(key)
-				return True
-
-		self.scope_user_email = pop("user:email")
-		self.scope_package = pop("package")
-		self.scope_package_release = pop("package:release") or self.scope_package
-		self.scope_package_screenshot = pop("package:screenshot") or self.scope_package
-		return v
-
 	def can_operate_on_package(self, package):
-		if (self.client is not None and
-				not (self.scope_package or self.scope_package_release or self.scope_package_screenshot)):
+		if self.client is not None:
 			return False
 
 		if self.package and self.package != package:
@@ -101,12 +71,13 @@ class AuditSeverity(enum.Enum):
 	def __str__(self):
 		return self.name
 
-	def get_title(self):
+	@property
+	def title(self):
 		return self.name.replace("_", " ").title()
 
 	@classmethod
 	def choices(cls):
-		return [(choice, choice.get_title()) for choice in cls]
+		return [(choice, choice.title) for choice in cls]
 
 	@classmethod
 	def coerce(cls, item):
@@ -172,6 +143,7 @@ class ForumTopic(db.Model):
 	author    = db.relationship("User", back_populates="forum_topics")
 
 	wip       = db.Column(db.Boolean, default=False, nullable=False)
+	# TODO: remove
 	discarded = db.Column(db.Boolean, default=False, nullable=False)
 
 	type      = db.Column(db.Enum(PackageType), nullable=False)
@@ -184,6 +156,10 @@ class ForumTopic(db.Model):
 
 	created_at = db.Column(db.DateTime, nullable=False, default=datetime.datetime.utcnow)
 
+	@property
+	def url(self):
+		return "https://forum.minetest.net/viewtopic.php?t=" + str(self.topic_id)
+
 	def get_repo_url(self):
 		if self.link is None:
 			return None
@@ -205,7 +181,6 @@ class ForumTopic(db.Model):
 			"posts":  self.posts,
 			"views":  self.views,
 			"is_wip": self.wip,
-			"discarded":  self.discarded,
 			"created_at": self.created_at.isoformat(),
 		}
 

app/models/collections.py

@@ -55,6 +55,7 @@ class Collection(db.Model):
 	long_description = db.Column(db.UnicodeText, nullable=True)
 	created_at = db.Column(db.DateTime, nullable=False, default=datetime.datetime.utcnow)
 	private = db.Column(db.Boolean, nullable=False, default=False)
+	pinned = db.Column(db.Boolean, nullable=False, default=False)
 
 	packages = db.relationship("Package", secondary=CollectionPackage.__table__, backref="collections")
 	items = db.relationship("CollectionPackage", back_populates="collection", order_by=db.asc("order"),
@@ -94,7 +95,7 @@ class Collection(db.Model):
 		elif type(perm) != Permission:
 			raise Exception("Unknown permission given to Collection.check_perm()")
 
-		if not user.is_authenticated:
+		if user is None or not user.is_authenticated:
 			return perm == Permission.VIEW_COLLECTION and not self.private
 
 		can_view = not self.private or self.author == user or user.rank.at_least(UserRank.MODERATOR)

app/models/packages.py

@@ -17,21 +17,23 @@
 
 import datetime
 import enum
+import os
 
+import typing
 from flask import url_for
-from flask_babel import lazy_gettext
-from flask_sqlalchemy import BaseQuery
-from sqlalchemy import or_
+from flask_babel import lazy_gettext, get_locale, gettext, pgettext
+from flask_sqlalchemy.query import Query
+from sqlalchemy import or_, func
+from sqlalchemy.dialects.postgresql import insert
 from sqlalchemy_searchable import SearchQueryMixin
 from sqlalchemy_utils.types import TSVectorType
-from sqlalchemy.dialects.postgresql import insert
 
+from app import app
 from . import db
 from .users import Permission, UserRank, User
-from app import app
 
 
-class PackageQuery(BaseQuery, SearchQueryMixin):
+class PackageQuery(Query, SearchQueryMixin):
 	pass
 
 
@@ -79,6 +81,42 @@ class PackageType(enum.Enum):
 		elif self == PackageType.TXP:
 			return lazy_gettext("Texture Packs")
 
+	def get_top_ordinal(self, place: int):
+		if place == 1:
+			if self == PackageType.MOD:
+				return lazy_gettext("Top mod")
+			elif self == PackageType.GAME:
+				return lazy_gettext("Top game")
+			elif self == PackageType.TXP:
+				return lazy_gettext("Top texture pack")
+		else:
+			if self == PackageType.MOD:
+				return lazy_gettext("Top %(place)d mod", place=place)
+			elif self == PackageType.GAME:
+				return lazy_gettext("Top %(place)d game", place=place)
+			elif self == PackageType.TXP:
+				return lazy_gettext("Top %(place)d texture pack", place=place)
+
+	def get_top_ordinal_description(self, display_name: str, place: int):
+		if self == PackageType.MOD:
+			return lazy_gettext(u"%(display_name)s has a mod placed at #%(place)d.",
+					display_name=display_name,  place=place)
+		elif self == PackageType.GAME:
+			return lazy_gettext(u"%(display_name)s has a game placed at #%(place)d.",
+					display_name=display_name,  place=place)
+		elif self == PackageType.TXP:
+			return lazy_gettext(u"%(display_name)s has a texture pack placed at #%(place)d.",
+					display_name=display_name,  place=place)
+
+	@property
+	def do_you_recommend(self):
+		if self == PackageType.MOD:
+			return lazy_gettext(u"Do you recommend this mod?")
+		elif self == PackageType.GAME:
+			return lazy_gettext(u"Do you recommend this game?")
+		elif self == PackageType.TXP:
+			return lazy_gettext(u"Do you recommend this texture pack?")
+
 	@classmethod
 	def get(cls, name):
 		try:
@@ -88,7 +126,7 @@ class PackageType(enum.Enum):
 
 	@classmethod
 	def choices(cls):
-		return [(choice, choice.text) for choice in cls]
+		return [(choice.name.lower(), choice.text) for choice in cls]
 
 	@classmethod
 	def coerce(cls, item):
@@ -97,7 +135,7 @@ class PackageType(enum.Enum):
 
 class PackageDevState(enum.Enum):
 	WIP = "Work in Progress"
-	BETA  = "Beta"
+	BETA = "Beta"
 	ACTIVELY_DEVELOPED = "Actively Developed"
 	MAINTENANCE_ONLY = "Maintenance Only"
 	AS_IS = "As-Is"
@@ -110,17 +148,41 @@ class PackageDevState(enum.Enum):
 	def __str__(self):
 		return self.name
 
+	@property
+	def title(self):
+		if self == PackageDevState.WIP:
+			# NOTE: Package maintenance state
+			return lazy_gettext("Looking for Maintainer")
+		elif self == PackageDevState.BETA:
+			# NOTE: Package maintenance state
+			return lazy_gettext("Beta")
+		elif self == PackageDevState.ACTIVELY_DEVELOPED:
+			# NOTE: Package maintenance state
+			return lazy_gettext("Actively Developed")
+		elif self == PackageDevState.MAINTENANCE_ONLY:
+			# NOTE: Package maintenance state
+			return lazy_gettext("Maintenance Only")
+		elif self == PackageDevState.AS_IS:
+			# NOTE: Package maintenance state
+			return lazy_gettext("As-is")
+		elif self == PackageDevState.DEPRECATED:
+			# NOTE: Package maintenance state
+			return lazy_gettext("Deprecated")
+		elif self == PackageDevState.LOOKING_FOR_MAINTAINER:
+			# NOTE: Package maintenance state
+			return lazy_gettext("Looking for Maintainer")
+
 	def get_desc(self):
 		if self == PackageDevState.WIP:
-			return "Under active development, and may break worlds/things without warning"
+			return lazy_gettext("Under active development, and may break worlds/things without warning")
 		elif self == PackageDevState.BETA:
-			return "Fully playable, but with some breakages/changes expected"
+			return lazy_gettext("Fully playable, but with some breakages/changes expected")
 		elif self == PackageDevState.MAINTENANCE_ONLY:
-			return "Finished, with bug fixes being made as needed"
+			return lazy_gettext("Finished, with bug fixes being made as needed")
 		elif self == PackageDevState.AS_IS:
-			return "Finished, the maintainer doesn't intend to continue working on it or provide support"
+			return lazy_gettext("Finished, the maintainer doesn't intend to continue working on it or provide support")
 		elif self == PackageDevState.DEPRECATED:
-			return "The maintainer doesn't recommend this package. See the description for more info"
+			return lazy_gettext("The maintainer doesn't recommend this package. See the description for more info")
 		else:
 			return None
 
@@ -206,15 +268,6 @@ class PackageState(enum.Enum):
 		return item if type(item) == PackageState else PackageState[item.upper()]
 
 
-PACKAGE_STATE_FLOW = {
-	PackageState.WIP: {PackageState.READY_FOR_REVIEW},
-	PackageState.CHANGES_NEEDED: {PackageState.READY_FOR_REVIEW},
-	PackageState.READY_FOR_REVIEW: {PackageState.WIP, PackageState.CHANGES_NEEDED, PackageState.APPROVED},
-	PackageState.APPROVED: {PackageState.CHANGES_NEEDED},
-	PackageState.DELETED: {PackageState.READY_FOR_REVIEW},
-}
-
-
 PackageProvides = db.Table("provides",
 	db.Column("package_id",    db.Integer, db.ForeignKey("package.id"), primary_key=True),
 	db.Column("metapackage_id", db.Integer, db.ForeignKey("meta_package.id"), primary_key=True)
@@ -336,12 +389,6 @@ class PackageGameSupport(db.Model):
 
 	__table_args__ = (db.UniqueConstraint("game_id", "package_id", name="_package_game_support_uc"),)
 
-	def __init__(self, package, game, confidence, supports):
-		self.package = package
-		self.game = game
-		self.confidence = confidence
-		self.supports = supports
-
 
 class Package(db.Model):
 	query_class  = PackageQuery
@@ -399,29 +446,41 @@ class Package(db.Model):
 	forums       = db.Column(db.Integer,     nullable=True)
 	video_url    = db.Column(db.String(200), nullable=True, default=None)
 	donate_url   = db.Column(db.String(200), nullable=True, default=None)
+	translation_url = db.Column(db.String(200), nullable=True)
 
 	@property
 	def donate_url_actual(self):
 		return self.donate_url or self.author.donate_url
 
+	@property
+	def forums_url(self) -> typing.Optional[str]:
+		if self.forums is None:
+			return None
+
+		return "https://forum.minetest.net/viewtopic.php?t=" + str(self.forums)
+
 	enable_game_support_detection = db.Column(db.Boolean, nullable=False, default=True)
 
+	translations = db.relationship("PackageTranslation", back_populates="package",
+			lazy="dynamic", order_by=db.asc("package_translation_language_id"),
+			cascade="all, delete, delete-orphan")
+
 	provides = db.relationship("MetaPackage", secondary=PackageProvides, order_by=db.asc("name"), back_populates="packages")
 
 	dependencies = db.relationship("Dependency", back_populates="depender", lazy="dynamic", foreign_keys=[Dependency.depender_id])
 
 	supported_games = db.relationship("PackageGameSupport", back_populates="package", lazy="dynamic",
-			foreign_keys=[PackageGameSupport.package_id])
+			foreign_keys=[PackageGameSupport.package_id], cascade="all, delete, delete-orphan")
 
 	game_supported_mods = db.relationship("PackageGameSupport", back_populates="game", lazy="dynamic",
-			foreign_keys=[PackageGameSupport.game_id])
+			foreign_keys=[PackageGameSupport.game_id], cascade="all, delete, delete-orphan")
 
 	tags = db.relationship("Tag", secondary=Tags, back_populates="packages")
 
 	content_warnings = db.relationship("ContentWarning", secondary=ContentWarnings, back_populates="packages")
 
 	releases = db.relationship("PackageRelease", back_populates="package",
-			lazy="dynamic", order_by=db.desc("package_release_releaseDate"), cascade="all, delete, delete-orphan")
+			lazy="dynamic", order_by=db.desc("package_release_created_at"), cascade="all, delete, delete-orphan")
 
 	screenshots = db.relationship("PackageScreenshot", back_populates="package", foreign_keys="PackageScreenshot.package_id",
 			lazy="dynamic", order_by=db.asc("package_screenshot_order"), cascade="all, delete, delete-orphan")
@@ -431,15 +490,15 @@ class Package(db.Model):
 			primaryjoin="and_(Package.id==PackageScreenshot.package_id, PackageScreenshot.approved)")
 
 	cover_image_id = db.Column(db.Integer, db.ForeignKey("package_screenshot.id"), nullable=True, default=None)
-	cover_image = db.relationship("PackageScreenshot", uselist=False, foreign_keys=[cover_image_id])
+	cover_image = db.relationship("PackageScreenshot", uselist=False, foreign_keys=[cover_image_id], post_update=True)
 
 	maintainers = db.relationship("User", secondary=maintainers)
 
 	threads = db.relationship("Thread", back_populates="package", order_by=db.desc("thread_created_at"),
 			foreign_keys="Thread.package_id", cascade="all, delete, delete-orphan", lazy="dynamic")
 
-	reviews = db.relationship("PackageReview", back_populates="package",
-			order_by=[db.desc("package_review_score"),db.desc("package_review_created_at")],
+	reviews = db.relationship("PackageReview", back_populates="package", lazy="dynamic",
+			order_by=[db.desc("package_review_score"), db.desc("package_review_created_at")],
 			cascade="all, delete, delete-orphan")
 
 	audit_log_entries = db.relationship("AuditLogEntry", foreign_keys="AuditLogEntry.package_id",
@@ -449,7 +508,7 @@ class Package(db.Model):
 			back_populates="package", cascade="all, delete, delete-orphan")
 
 	tokens = db.relationship("APIToken", foreign_keys="APIToken.package_id", back_populates="package",
-			cascade="all, delete, delete-orphan")
+			cascade="all, delete")
 
 	update_config = db.relationship("PackageUpdateConfig", uselist=False, back_populates="package",
 			cascade="all, delete, delete-orphan")
@@ -480,13 +539,44 @@ class Package(db.Model):
 		if name.endswith("_game"):
 			name = name[:-5]
 
-		return Package.query.filter(
-			or_(Package.name == name, Package.name == name + "_game"),
+		return Package.query.filter(or_(Package.name == name, Package.name == name + "_game"),
 			Package.author.has(username=parts[0])).first()
 
 	def get_id(self):
 		return "{}/{}".format(self.author.username, self.name)
 
+	@property
+	def normalised_name(self):
+		name = self.name
+		if name.endswith("_game"):
+			name = name[:-5]
+		return name
+
+	def get_translated(self, lang=None, load_desc=True):
+		if lang is None:
+			locale = get_locale()
+			if locale:
+				lang = locale.language
+			else:
+				lang = "en"
+
+		translation: typing.Optional[PackageTranslation] = None
+		if lang != "en":
+			translation = self.translations.filter_by(language_id=lang).first()
+
+		if translation is None:
+			return {
+				"title": self.title,
+				"short_desc": self.short_desc,
+				"desc": self.desc if load_desc else None,
+			}
+
+		return {
+			"title": translation.title or self.title,
+			"short_desc": translation.short_desc or self.short_desc,
+			"desc": (translation.desc or self.desc) if load_desc else None,
+		}
+
 	def get_sorted_dependencies(self, is_hard=None):
 		query = self.dependencies
 		if is_hard is not None:
@@ -502,14 +592,14 @@ class Package(db.Model):
 	def get_sorted_optional_dependencies(self):
 		return self.get_sorted_dependencies(False)
 
-	def get_sorted_game_support(self):
+	def get_sorted_game_support(self) -> list[PackageGameSupport]:
 		query = self.supported_games.filter(PackageGameSupport.game.has(state=PackageState.APPROVED))
 
 		supported = query.all()
 		supported.sort(key=lambda x: -(x.game.score + 100000*x.confidence))
 		return supported
 
-	def get_sorted_game_support_pair(self):
+	def get_sorted_game_support_pair(self) -> list[list[PackageGameSupport]]:
 		supported = self.get_sorted_game_support()
 		return [
 			[x for x in supported if x.supports],
@@ -527,20 +617,21 @@ class Package(db.Model):
 			"type": self.type.to_name(),
 		}
 
-	def as_short_dict(self, base_url, version=None, release_id=None, no_load=False):
-		tnurl = self.get_thumb_url(1)
+	def as_short_dict(self, base_url, version=None, release_id=None, no_load=False, lang="en", include_vcs=False):
+		tnurl = self.get_thumb_url(1, format="png")
 
 		if release_id is None and no_load == False:
 			release = self.get_download_release(version=version)
 			release_id = release and release.id
 
-		short_desc = self.short_desc
+		meta = self.get_translated(lang, load_desc=False)
+		short_desc = meta["short_desc"]
 		if self.dev_state == PackageDevState.WIP:
-			short_desc = "Work in Progress. " + self.short_desc
+			short_desc = gettext("Work in Progress") + ". " + self.short_desc
 
 		ret = {
 			"name": self.name,
-			"title": self.title,
+			"title": meta["title"],
 			"author": self.author.username,
 			"short_description": short_desc,
 			"type": self.type.to_name(),
@@ -552,11 +643,21 @@ class Package(db.Model):
 		if not ret["aliases"]:
 			del ret["aliases"]
 
+		if include_vcs:
+			ret["repo"] = self.repo
+
 		return ret
 
-	def as_dict(self, base_url, version=None):
-		tnurl = self.get_thumb_url(1)
+	def as_dict(self, base_url, version=None, lang="en", screenshots_dict=False):
+		tnurl = self.get_thumb_url(1, format="png")
 		release = self.get_download_release(version=version)
+		meta = self.get_translated(lang)
+
+		if screenshots_dict:
+			screenshots = [ss.as_short_dict(base_url) for ss in self.screenshots]
+		else:
+			screenshots = [base_url + ss.url for ss in self.screenshots]
+
 		return {
 			"author": self.author.username,
 			"maintainers": [x.username for x in self.maintainers],
@@ -565,9 +666,9 @@ class Package(db.Model):
 			"dev_state": self.dev_state.name if self.dev_state else None,
 
 			"name": self.name,
-			"title": self.title,
-			"short_description": self.short_desc,
-			"long_description": self.desc,
+			"title": meta["title"],
+			"short_description": meta["short_desc"],
+			"long_description": meta["desc"],
 			"type": self.type.to_name(),
 			"created_at": self.created_at.isoformat(),
 
@@ -579,14 +680,16 @@ class Package(db.Model):
 			"issue_tracker": self.issueTracker,
 			"forums": self.forums,
 			"video_url": self.video_url,
+			"video_thumbnail_url": self.get_video_thumbnail_url(True),
 			"donate_url": self.donate_url_actual,
+			"translation_url": self.translation_url,
 
 			"tags": sorted([x.name for x in self.tags]),
 			"content_warnings": sorted([x.name for x in self.content_warnings]),
 
 			"provides": sorted([x.name for x in self.provides]),
 			"thumbnail": (base_url + tnurl) if tnurl is not None else None,
-			"screenshots": [base_url + ss.url for ss in self.screenshots],
+			"screenshots": screenshots,
 
 			"url": base_url + self.get_url("packages.download"),
 			"release": release and release.id,
@@ -603,21 +706,21 @@ class Package(db.Model):
 			]
 		}
 
-	def get_thumb_or_placeholder(self, level=2):
-		return self.get_thumb_url(level) or "/static/placeholder.png"
+	def get_thumb_or_placeholder(self, level=2, format="webp"):
+		return self.get_thumb_url(level, False, format) or "/static/placeholder.png"
 
-	def get_thumb_url(self, level=2, abs=False):
+	def get_thumb_url(self, level=2, abs=False, format="webp"):
 		screenshot = self.main_screenshot
-		url = screenshot.get_thumb_url(level) if screenshot is not None else None
+		url = screenshot.get_thumb_url(level, format) if screenshot is not None else None
 		if abs:
 			from app.utils import abs_url
 			return abs_url(url)
 		else:
 			return url
 
-	def get_cover_image_url(self):
+	def get_cover_image_url(self, format="webp"):
 		screenshot = self.cover_image or self.main_screenshot
-		return screenshot and screenshot.get_thumb_url(4)
+		return screenshot and screenshot.get_thumb_url(4, format)
 
 	def get_url(self, endpoint, absolute=False, **kwargs):
 		if absolute:
@@ -635,16 +738,32 @@ class Package(db.Model):
 		return "[![ContentDB]({})]({})" \
 			.format(self.get_shield_url(type), self.get_url("packages.view", True))
 
+	def get_video_thumbnail_url(self, absolute: bool = False):
+		from app.utils.url import get_youtube_id
+
+		if self.video_url is None:
+			return None
+
+		id_ = get_youtube_id(self.video_url)
+		if id_ is None:
+			return None
+
+		if absolute:
+			from app.utils import abs_url_for
+			return abs_url_for("thumbnails.youtube", id_=id_)
+		else:
+			return url_for("thumbnails.youtube", id_=id_)
+
 	def get_set_state_url(self, state):
 		if type(state) == str:
 			state = PackageState[state]
 		elif type(state) != PackageState:
-			raise Exception("Unknown state given to Package.can_move_to_state()")
+			raise Exception("Unknown state given to Package.get_set_state_url()")
 
 		return url_for("packages.move_to_state",
 				author=self.author.username, name=self.name, state=state.name.lower())
 
-	def get_download_release(self, version=None):
+	def get_download_release(self, version=None) -> typing.Optional["PackageRelease"]:
 		for rel in self.releases:
 			if rel.approved and (version is None or
 					((rel.min_rel is None or rel.min_rel_id <= version.id) and
@@ -715,66 +834,35 @@ class Package(db.Model):
 	def get_missing_hard_dependencies(self):
 		return [mp.name for mp in self.get_missing_hard_dependencies_query().all()]
 
-	def can_move_to_state(self, user, state):
-		if not user.is_authenticated:
-			return False
-
-		if type(state) == str:
-			state = PackageState[state]
-		elif type(state) != PackageState:
-			raise Exception("Unknown state given to Package.can_move_to_state()")
-
-		if state not in PACKAGE_STATE_FLOW[self.state]:
-			return False
-
-		if state == PackageState.READY_FOR_REVIEW or state == PackageState.APPROVED:
-			if state == PackageState.APPROVED and not self.check_perm(user, Permission.APPROVE_NEW):
-				return False
-
-			if not (self.check_perm(user, Permission.APPROVE_NEW) or self.check_perm(user, Permission.EDIT_PACKAGE)):
-				return False
-
-			if state == PackageState.APPROVED and ("Other" in self.license.name or "Other" in self.media_license.name):
-				return False
-
-			if self.get_missing_hard_dependencies_query().count() > 0:
-				return False
-
-			needs_screenshot = \
-				(self.type == self.type.GAME or self.type == self.type.TXP) and self.screenshots.count() == 0
-
-			return self.releases.filter(PackageRelease.task_id==None).count() > 0 and not needs_screenshot
-
-		elif state == PackageState.CHANGES_NEEDED:
-			return self.check_perm(user, Permission.APPROVE_NEW)
-
-		elif state == PackageState.WIP:
-			return self.check_perm(user, Permission.EDIT_PACKAGE) and \
-				(user in self.maintainers or user.rank.at_least(UserRank.ADMIN))
-
-		return True
-
 	def get_next_states(self, user):
+		from app.logic.package_approval import can_move_to_state
+
 		states = []
 
 		for state in PackageState:
-			if self.can_move_to_state(user, state):
+			if can_move_to_state(self, user, state):
 				states.append(state)
 
 		return states
 
 	def as_score_dict(self):
+		reviews = self.get_review_summary()
 		return {
 			"author": self.author.username,
 			"name": self.name,
 			"score": self.score,
 			"score_downloads": self.score_downloads,
 			"score_reviews": self.score - self.score_downloads,
-			"downloads": self.downloads
+			"downloads": self.downloads,
+			"reviews": {
+				"positive": reviews[0],
+				"neutral": reviews[1],
+				"negative": reviews[2],
+			},
 		}
 
 	def recalculate_score(self):
-		review_scores = [ 100 * r.as_weight() for r in self.reviews ]
+		review_scores = [ 150 * r.as_weight() for r in self.reviews ]
 		self.score = self.score_downloads + sum(review_scores)
 
 	def get_conf_file_name(self):
@@ -785,6 +873,57 @@ class Package(db.Model):
 		elif self.type == PackageType.GAME:
 			return "game.conf"
 
+	def get_review_summary(self):
+		from app.models import PackageReview
+		rows = (db.session.query(PackageReview.rating, func.count(PackageReview.id))
+			.select_from(PackageReview)
+			.where(PackageReview.package_id == self.id)
+			.group_by(PackageReview.rating)
+			.all())
+
+		negative = 0
+		neutral = 0
+		positive = 0
+		for rating, count in rows:
+			if rating > 3:
+				positive += count
+			elif rating == 3:
+				neutral += count
+			else:
+				negative += count
+
+		return [positive, neutral, negative]
+
+
+class Language(db.Model):
+	id = db.Column(db.String(10), primary_key=True)
+	title = db.Column(db.String(100), unique=True, nullable=False)
+
+	packages = db.relationship("Package", secondary="package_translation", lazy="dynamic")
+
+	@property
+	def has_contentdb_translation(self):
+		return self.id in app.config["LANGUAGES"].keys()
+
+	def as_dict(self):
+		return {
+			"id": self.id,
+			"title": self.title,
+			"has_contentdb_translation": self.has_contentdb_translation,
+		}
+
+
+class PackageTranslation(db.Model):
+	package_id = db.Column(db.Integer, db.ForeignKey("package.id"), primary_key=True)
+	package = db.relationship("Package", back_populates="translations", foreign_keys=[package_id])
+
+	language_id = db.Column(db.String(10), db.ForeignKey("language.id"), primary_key=True)
+	language = db.relationship("Language", foreign_keys=[language_id])
+
+	title = db.Column(db.Unicode(100), nullable=True)
+	short_desc = db.Column(db.Unicode(200), nullable=True)
+	desc = db.Column(db.UnicodeText, nullable=True)
+
 
 class MetaPackage(db.Model):
 	id = db.Column(db.Integer, primary_key=True)
@@ -854,6 +993,13 @@ class ContentWarning(db.Model):
 		regex = re.compile("[^a-z_]")
 		self.name = regex.sub("", self.title.lower().replace(" ", "_"))
 
+	def get_translated(self):
+		# Translations are automated on dynamic data using `extract_translations.py`
+		return {
+			"title": pgettext("tags", self.title),
+			"description": pgettext("content_warnings", self.description),
+		}
+
 	def as_dict(self):
 		description = self.description if self.description != "" else None
 		return { "name": self.name, "title": self.title, "description": description }
@@ -879,6 +1025,13 @@ class Tag(db.Model):
 		regex = re.compile("[^a-z_]")
 		self.name = regex.sub("", self.title.lower().replace(" ", "_"))
 
+	def get_translated(self):
+		# Translations are automated on dynamic data using `extract_translations.py`
+		return {
+			"title": pgettext("tags", self.title),
+			"description": pgettext("tags", self.description) if self.description else "",
+		}
+
 	def as_dict(self):
 		description = self.description if self.description != "" else None
 		return {
@@ -898,6 +1051,10 @@ class MinetestRelease(db.Model):
 		self.name = name
 		self.protocol = protocol
 
+	@property
+	def value(self):
+		return self.name
+
 	def get_actual(self):
 		return None if self.name == "None" else self
 
@@ -909,7 +1066,7 @@ class MinetestRelease(db.Model):
 		}
 
 	@classmethod
-	def get(cls, version, protocol_num):
+	def get(cls, version: typing.Optional[str], protocol_num: typing.Optional[str]) -> typing.Optional["MinetestRelease"]:
 		if version:
 			parts = version.strip().split(".")
 			if len(parts) >= 2:
@@ -937,13 +1094,24 @@ class PackageRelease(db.Model):
 	package_id   = db.Column(db.Integer, db.ForeignKey("package.id"))
 	package      = db.relationship("Package", back_populates="releases", foreign_keys=[package_id])
 
+	name         = db.Column(db.String(30), nullable=False)
 	title        = db.Column(db.String(100), nullable=False)
-	releaseDate  = db.Column(db.DateTime,    nullable=False)
+	created_at  = db.Column(db.DateTime,    nullable=False)
 	url          = db.Column(db.String(200), nullable=False, default="")
 	approved     = db.Column(db.Boolean, nullable=False, default=False)
 	task_id      = db.Column(db.String(37), nullable=True)
 	commit_hash  = db.Column(db.String(41), nullable=True, default=None)
 	downloads    = db.Column(db.Integer, nullable=False, default=0)
+	release_notes = db.Column(db.UnicodeText, nullable=True, default=None)
+
+	@property
+	def summary(self) -> str:
+		if self.release_notes is None or \
+				self.release_notes.startswith("-") or \
+				self.release_notes.startswith("*"):
+			return self.title
+
+		return self.release_notes.split("\n")[0]
 
 	min_rel_id = db.Column(db.Integer, db.ForeignKey("minetest_release.id"), nullable=True, server_default=None)
 	min_rel    = db.relationship("MinetestRelease", foreign_keys=[min_rel_id])
@@ -958,16 +1126,36 @@ class PackageRelease(db.Model):
 	def file_path(self):
 		return self.url.replace("/uploads/", app.config["UPLOAD_DIR"])
 
+	@property
+	def file_size_bytes(self):
+		path = self.file_path
+		if not os.path.isfile(path):
+			return 0
+
+		file_stats = os.stat(path)
+		return file_stats.st_size
+
+	@property
+	def file_size(self):
+		size = self.file_size_bytes / 1024
+		if size > 1024:
+			return f"{round(size / 1024, 1)} MB"
+		else:
+			return f"{round(size)} KB"
+
 	def as_dict(self):
 		return {
 			"id": self.id,
+			"name": self.name,
 			"title": self.title,
+			"release_notes": self.release_notes,
 			"url": self.url if self.url != "" else None,
-			"release_date": self.releaseDate.isoformat(),
+			"release_date": self.created_at.isoformat(),
 			"commit": self.commit_hash,
 			"downloads": self.downloads,
 			"min_minetest_version": self.min_rel and self.min_rel.as_dict(),
 			"max_minetest_version": self.max_rel and self.max_rel.as_dict(),
+			"size": self.file_size_bytes,
 		}
 
 	def as_long_dict(self):
@@ -975,12 +1163,13 @@ class PackageRelease(db.Model):
 			"id": self.id,
 			"title": self.title,
 			"url": self.url if self.url != "" else None,
-			"release_date": self.releaseDate.isoformat(),
+			"release_date": self.created_at.isoformat(),
 			"commit": self.commit_hash,
 			"downloads": self.downloads,
 			"min_minetest_version": self.min_rel and self.min_rel.as_dict(),
 			"max_minetest_version": self.max_rel and self.max_rel.as_dict(),
-			"package": self.package.as_key_dict()
+			"package": self.package.as_key_dict(),
+			"size": self.file_size_bytes,
 		}
 
 	def get_edit_url(self):
@@ -1002,7 +1191,7 @@ class PackageRelease(db.Model):
 				id=self.id)
 
 	def __init__(self):
-		self.releaseDate = datetime.datetime.now()
+		self.created_at = datetime.datetime.now()
 
 	def get_download_filename(self):
 		return f"{self.package.name}_{self.id}.zip"
@@ -1025,7 +1214,7 @@ class PackageRelease(db.Model):
 		return True
 
 	def check_perm(self, user, perm):
-		if not user.is_authenticated:
+		if not hasattr(user, "rank") or user.is_banned:
 			return False
 
 		if type(perm) == str:
@@ -1051,9 +1240,7 @@ class PackageRelease(db.Model):
 
 			return count > 0
 		elif perm == Permission.APPROVE_RELEASE:
-			return user.rank.at_least(UserRank.APPROVER) or \
-					(is_maintainer and user.rank.at_least(
-						UserRank.MEMBER if self.approved else UserRank.NEW_MEMBER))
+			return is_maintainer or user.rank.at_least(UserRank.APPROVER)
 		else:
 			raise Exception("Permission {} is not related to releases".format(perm.name))
 
@@ -1089,6 +1276,23 @@ class PackageScreenshot(db.Model):
 	def file_path(self):
 		return self.url.replace("/uploads/", app.config["UPLOAD_DIR"])
 
+	@property
+	def file_size_bytes(self):
+		path = self.file_path
+		if not os.path.isfile(path):
+			return 0
+
+		file_stats = os.stat(path)
+		return file_stats.st_size
+
+	@property
+	def file_size(self):
+		size = self.file_size_bytes / 1024
+		if size > 1024:
+			return f"{round(size / 1024, 1)} MB"
+		else:
+			return f"{round(size)} KB"
+
 	def get_edit_url(self):
 		return url_for("packages.edit_screenshot",
 				author=self.package.author.username,
@@ -1101,8 +1305,12 @@ class PackageScreenshot(db.Model):
 				name=self.package.name,
 				id=self.id)
 
-	def get_thumb_url(self, level=2):
-		return self.url.replace("/uploads/", "/thumbnails/{:d}/".format(level))
+	def get_thumb_url(self, level=2, format="webp"):
+		url = self.url.replace("/uploads/", "/thumbnails/{:d}/".format(level))
+		if format is not None:
+			start = url[:url.rfind(".")]
+			url = f"{start}.{format}"
+		return url
 
 	def as_dict(self, base_url=""):
 		return {
@@ -1117,6 +1325,12 @@ class PackageScreenshot(db.Model):
 			"is_cover_image": self.package.cover_image == self,
 		}
 
+	def as_short_dict(self, base_url=""):
+		return {
+			"title": self.title,
+			"url": base_url + self.url,
+		}
+
 
 class PackageUpdateTrigger(enum.Enum):
 	COMMIT = "New Commit"
@@ -1167,25 +1381,27 @@ class PackageUpdateConfig(db.Model):
 
 	def get_message(self):
 		if self.trigger == PackageUpdateTrigger.COMMIT:
-			msg = "New commit {} found on the Git repo.".format(self.last_commit[0:5])
+			msg = lazy_gettext("New commit %(hash)s found on the Git repo.", hash=self.last_commit[0:5])
 
 			last_release = self.package.releases.first()
 			if last_release and last_release.commit_hash:
-				msg += " The last release was commit {}".format(last_release.commit_hash[0:5])
+				msg += " " + lazy_gettext("The last release was commit %(hash)s",
+						hash=last_release.commit_hash[0:5])
 
 			return msg
 
 		else:
-			return "New tag {} found on the Git repo.".format(self.last_tag)
+			return lazy_gettext("New tag %(tag_name)s found on the Git repo.", tag_name=self.last_tag)
 
-	def get_title(self):
+	@property
+	def title(self):
 		return self.last_tag or self.outdated_at.strftime("%Y-%m-%d")
 
 	def get_ref(self):
 		return self.last_tag or self.last_commit
 
 	def get_create_release_url(self):
-		return self.package.get_url("packages.create_release", title=self.get_title(), ref=self.get_ref())
+		return self.package.get_url("packages.create_release", title=self.title, ref=self.get_ref())
 
 
 class PackageAlias(db.Model):

app/models/threads.py

@@ -18,6 +18,8 @@ import datetime
 from typing import Tuple, List
 
 from flask import url_for
+from sqlalchemy import select, func, text
+from sqlalchemy.orm import column_property
 
 from . import db
 from .users import Permission, UserRank, User
@@ -59,6 +61,11 @@ class Thread(db.Model):
 			lazy=True, order_by=db.asc("id"), viewonly=True,
 			primaryjoin="Thread.id==ThreadReply.thread_id")
 
+	replies_count = column_property(select(func.count(text("thread_reply.id")))
+			.select_from(text("thread_reply"))
+			.where(text("thread_reply.thread_id") == id)
+			.as_scalar())
+
 	def get_description(self):
 		comment = self.first_reply.comment.replace("\r\n", " ").replace("\n", " ").replace("  ", " ")
 		if len(comment) > 100:
@@ -169,7 +176,7 @@ class ThreadReply(db.Model):
 class PackageReview(db.Model):
 	id         = db.Column(db.Integer, primary_key=True)
 
-	package_id = db.Column(db.Integer, db.ForeignKey("package.id"), nullable=True)
+	package_id = db.Column(db.Integer, db.ForeignKey("package.id"), nullable=False)
 	package    = db.relationship("Package", foreign_keys=[package_id], back_populates="reviews")
 
 	created_at = db.Column(db.DateTime, nullable=False, default=datetime.datetime.utcnow)
@@ -177,6 +184,9 @@ class PackageReview(db.Model):
 	author_id  = db.Column(db.Integer, db.ForeignKey("user.id"), nullable=False)
 	author     = db.relationship("User", foreign_keys=[author_id], back_populates="reviews")
 
+	language_id = db.Column(db.String, db.ForeignKey("language.id"), nullable=True, default=None)
+	language    = db.relationship("Language", foreign_keys=[language_id])
+
 	rating     = db.Column(db.Integer, nullable=False)
 
 	thread     = db.relationship("Thread", uselist=False, back_populates="review")

app/models/users.py

@@ -17,11 +17,11 @@
 import datetime
 import enum
 
-from flask import url_for
+from flask import current_app
+from flask_babel import lazy_gettext
 from flask_login import UserMixin
 from sqlalchemy import desc, text
 
-from app import gravatar
 from . import db
 
 
@@ -40,8 +40,28 @@ class UserRank(enum.Enum):
 	def at_least(self, min):
 		return self.value >= min.value
 
-	def get_title(self):
-		return self.name.replace("_", " ").title()
+	@property
+	def title(self):
+		if self == UserRank.BANNED:
+			return lazy_gettext("Banned")
+		elif self == UserRank.NOT_JOINED:
+			return lazy_gettext("Not Joined")
+		elif self == UserRank.NEW_MEMBER:
+			return lazy_gettext("New Member")
+		elif self == UserRank.MEMBER:
+			return lazy_gettext("Member")
+		elif self == UserRank.TRUSTED_MEMBER:
+			return lazy_gettext("Trusted Member")
+		elif self == UserRank.APPROVER:
+			return lazy_gettext("Approver")
+		elif self == UserRank.EDITOR:
+			return lazy_gettext("Editor")
+		elif self == UserRank.BOT:
+			return lazy_gettext("Bot")
+		elif self == UserRank.MODERATOR:
+			return lazy_gettext("Moderator")
+		elif self == UserRank.ADMIN:
+			return lazy_gettext("Admin")
 
 	def to_name(self):
 		return self.name.lower()
@@ -51,7 +71,7 @@ class UserRank(enum.Enum):
 
 	@classmethod
 	def choices(cls):
-		return [(choice, choice.get_title()) for choice in cls]
+		return [(choice, choice.title) for choice in cls]
 
 	@classmethod
 	def coerce(cls, item):
@@ -144,6 +164,7 @@ class User(db.Model, UserMixin):
 
 	# Account linking
 	github_username = db.Column(db.String(50, collation="NOCASE"), nullable=True, unique=True)
+	github_user_id = db.Column(db.Integer, nullable=True, unique=True)
 	forums_username = db.Column(db.String(50, collation="NOCASE"), nullable=True, unique=True)
 
 	# Access token for webhook setup
@@ -192,6 +213,10 @@ class User(db.Model, UserMixin):
 
 	ban = db.relationship("UserBan", foreign_keys="UserBan.user_id", back_populates="user", uselist=False)
 
+	@property
+	def is_banned(self):
+		return (self.ban and not self.ban.has_expired) or self.rank == UserRank.BANNED
+
 	def get_dict(self):
 		from app.utils.flask import abs_url_for
 		return {
@@ -222,13 +247,20 @@ class User(db.Model, UserMixin):
 	def can_access_todo_list(self):
 		return Permission.APPROVE_NEW.check(self) or Permission.APPROVE_RELEASE.check(self)
 
-	def get_profile_pic_url(self):
+	def get_profile_pic_url(self, absolute: bool = False):
 		if self.profile_pic:
-			return self.profile_pic
+			if absolute:
+				return current_app.config["BASE_URL"] + self.profile_pic
+			else:
+				return self.profile_pic
 		elif self.rank == UserRank.BOT:
-			return "/static/bot_avatar.png"
+			if absolute:
+				return current_app.config["BASE_URL"] + "/static/bot_avatar.png"
+			else:
+				return "/static/bot_avatar.png"
 		else:
-			return gravatar(self.email or f"{self.username}@content.minetest.net")
+			from app.utils.gravatar import get_gravatar
+			return get_gravatar(self.email or f"{self.username}@content.minetest.net")
 
 	def check_perm(self, user, perm):
 		if not user.is_authenticated:
@@ -271,12 +303,12 @@ class User(db.Model, UserMixin):
 
 		one_min_ago = datetime.datetime.utcnow() - datetime.timedelta(minutes=1)
 		if ThreadReply.query.filter_by(author=self) \
-				.filter(ThreadReply.created_at > one_min_ago).count() >= 2 * factor:
+				.filter(ThreadReply.created_at > one_min_ago, ThreadReply.is_status_update == False).count() >= 2 * factor:
 			return False
 
 		hour_ago = datetime.datetime.utcnow() - datetime.timedelta(hours=1)
 		if ThreadReply.query.filter_by(author=self) \
-				.filter(ThreadReply.created_at > hour_ago).count() >= 10 * factor:
+				.filter(ThreadReply.created_at > hour_ago, ThreadReply.is_status_update == False).count() >= 10 * factor:
 			return False
 
 		return True
@@ -318,7 +350,8 @@ class User(db.Model, UserMixin):
 		if other is None:
 			return False
 
-		if not self.is_authenticated or not other.is_authenticated:
+		# Anonymous users
+		if not hasattr(self, "id") or not hasattr(other, "id"):
 			return False
 
 		assert self.id > 0
@@ -345,6 +378,12 @@ class UserEmailVerification(db.Model):
 	is_password_reset = db.Column(db.Boolean, nullable=False, default=False)
 	created_at = db.Column(db.DateTime, nullable=False, default=datetime.datetime.utcnow)
 
+	@property
+	def is_expired(self):
+		delta = (datetime.datetime.now() - self.created_at)
+		delta: datetime.timedelta
+		return delta.total_seconds() > 12 * 60 * 60
+
 
 class EmailSubscription(db.Model):
 	id          = db.Column(db.Integer, primary_key=True)
@@ -394,36 +433,93 @@ class NotificationType(enum.Enum):
 	# Any other
 	OTHER          = 0
 
-
-	def get_title(self):
-		return self.name.replace("_", " ").title()
+	@property
+	def title(self):
+		if self == NotificationType.PACKAGE_EDIT:
+			# NOTE: PACKAGE_EDIT notification type
+			return lazy_gettext("Package Edit")
+		elif self == NotificationType.PACKAGE_APPROVAL:
+			# NOTE: PACKAGE_APPROVAL notification type
+			return lazy_gettext("Package Approval")
+		elif self == NotificationType.NEW_THREAD:
+			# NOTE: NEW_THREAD notification type
+			return lazy_gettext("New Thread")
+		elif self == NotificationType.NEW_REVIEW:
+			# NOTE: NEW_REVIEW notification type
+			return lazy_gettext("New Review")
+		elif self == NotificationType.THREAD_REPLY:
+			# NOTE: THREAD_REPLY notification type
+			return lazy_gettext("Thread Reply")
+		elif self == NotificationType.BOT:
+			# NOTE: BOT notification type
+			return lazy_gettext("Bot")
+		elif self == NotificationType.MAINTAINER:
+			# NOTE: MAINTAINER notification type
+			return lazy_gettext("Maintainer")
+		elif self == NotificationType.EDITOR_ALERT:
+			# NOTE: EDITOR_ALERT notification type
+			return lazy_gettext("Editor Alert")
+		elif self == NotificationType.EDITOR_MISC:
+			# NOTE: EDITOR_MISC notification type
+			return lazy_gettext("Editor Misc")
+		elif self == NotificationType.OTHER:
+			# NOTE: OTHER notification type
+			return lazy_gettext("Other")
+		else:
+			raise "Unknown notification type"
 
 	def to_name(self):
 		return self.name.lower()
 
-	def get_description(self):
+	@property
+	def this_is(self):
 		if self == NotificationType.PACKAGE_EDIT:
-			return "When another user edits your packages, releases, etc."
+			return lazy_gettext("This is a Package Edit notification.")
 		elif self == NotificationType.PACKAGE_APPROVAL:
-			return "Notifications from editors related to the package approval process."
+			return lazy_gettext("This is a Package Approval notification.")
 		elif self == NotificationType.NEW_THREAD:
-			return "When a thread is created on your package."
+			return lazy_gettext("This is a New Thread notification.")
 		elif self == NotificationType.NEW_REVIEW:
-			return "When a user posts a review on your package."
+			return lazy_gettext("This is a New Review notification.")
 		elif self == NotificationType.THREAD_REPLY:
-			return "When someone replies to a thread you're watching."
+			return lazy_gettext("This is a Thread Reply notification.")
 		elif self == NotificationType.BOT:
-			return "From a bot - for example, update notifications."
+			return lazy_gettext("This is a Bot notification.")
 		elif self == NotificationType.MAINTAINER:
-			return "When your package's maintainers change."
+			return lazy_gettext("This is a Maintainer change notification.")
 		elif self == NotificationType.EDITOR_ALERT:
-			return "For editors: Important alerts."
+			return lazy_gettext("This is an Editor Alert notification.")
 		elif self == NotificationType.EDITOR_MISC:
-			return "For editors: Minor notifications, including new threads."
+			return lazy_gettext("This is an Editor Misc notification.")
 		elif self == NotificationType.OTHER:
-			return "Minor notifications not important enough for a dedicated category."
+			return lazy_gettext("This is an Other notification.")
 		else:
-			return ""
+			raise "Unknown notification type"
+
+	@property
+	def description(self):
+		if self == NotificationType.PACKAGE_EDIT:
+			return lazy_gettext("When another user edits your packages, releases, etc.")
+		elif self == NotificationType.PACKAGE_APPROVAL:
+			return lazy_gettext("Notifications from editors related to the package approval process.")
+		elif self == NotificationType.NEW_THREAD:
+			return lazy_gettext("When a thread is created on your package.")
+		elif self == NotificationType.NEW_REVIEW:
+			return lazy_gettext("When a user posts a review on your package.")
+		elif self == NotificationType.THREAD_REPLY:
+			return lazy_gettext("When someone replies to a thread you're watching.")
+		elif self == NotificationType.BOT:
+			return lazy_gettext("From a bot - for example, update notifications.")
+		elif self == NotificationType.MAINTAINER:
+			return lazy_gettext("When your package's maintainers change.")
+		elif self == NotificationType.EDITOR_ALERT:
+			return lazy_gettext("For editors: Important alerts.")
+		elif self == NotificationType.EDITOR_MISC:
+			return lazy_gettext("For editors: Minor notifications, including new threads.")
+		elif self == NotificationType.OTHER:
+			return lazy_gettext("Minor notifications not important enough for a dedicated category.")
+		else:
+			raise "Unknown notification type"
 
 	def __str__(self):
 		return self.name
@@ -433,7 +529,7 @@ class NotificationType(enum.Enum):
 
 	@classmethod
 	def choices(cls):
-		return [(choice, choice.get_title()) for choice in cls]
+		return [(choice, choice.title) for choice in cls]
 
 	@classmethod
 	def coerce(cls, item):
@@ -560,6 +656,7 @@ class OAuthClient(db.Model):
 	redirect_url = db.Column(db.String(128), nullable=False)
 	approved = db.Column(db.Boolean, nullable=False, default=False)
 	verified = db.Column(db.Boolean, nullable=False, default=False)
+	is_clientside = db.Column(db.Boolean, nullable=False, default=False)
 
 	owner_id = db.Column(db.Integer, db.ForeignKey('user.id'), nullable=False)
 	owner = db.relationship("User", foreign_keys=[owner_id], back_populates="clients")
@@ -567,3 +664,11 @@ class OAuthClient(db.Model):
 	tokens = db.relationship("APIToken", back_populates="client", lazy="dynamic", cascade="all, delete, delete-orphan")
 
 	created_at = db.Column(db.DateTime, nullable=False, default=datetime.datetime.utcnow)
+
+	def get_app_type(self):
+		return "client" if self.is_clientside else "server"
+
+	def set_app_type(self, value):
+		self.is_clientside = value == "client"
+
+	app_type = property(get_app_type, set_app_type)

app/public/static/js/collection_editor.js

@@ -5,10 +5,10 @@
 
 
 function updateOrder() {
-	const elements = [...document.querySelector(".sortable").children];
+	const elements = [...document.querySelector("#package_list").children];
 	const ids = elements
 		.filter(x => !x.classList.contains("d-none"))
-		.map(x => x.dataset.id)
+		.map(x => x.dataset.id?.trim())
 		.filter(x => x);
 
 	document.querySelector("input[name='order']").value = ids.join(",");

app/public/static/js/email_disable_all.js

@@ -0,0 +1,10 @@
+// @author rubenwardy
+// @license magnet:?xt=urn:btih:1f739d935676111cfff4b4693e3816e664797050&dn=gpl-3.0.txt GPL-v3-or-Later
+
+"use strict";
+
+const disableAll = document.getElementById("disable-all");
+disableAll.classList.remove("d-none");
+disableAll.addEventListener("click", () => {
+	document.querySelectorAll("input[type='checkbox']").forEach(x => { x.checked = false; });
+});

app/public/static/js/package_edit.js

@@ -40,45 +40,56 @@ window.addEventListener("load", () => {
 		window.open("https://forum.minetest.net/viewtopic.php?t=" + forumsField.value, "_blank");
 	});
 
-	let hint = null;
-	function showHint(ele, text) {
-		if (hint) {
-			hint.remove();
+	function setupHints(id, hints) {
+		function onChange(val) {
+			val = val.toLowerCase();
+			Object.entries(hints).forEach(([key, func]) => {
+				if (func(val)) {
+					document.getElementById(key).classList.remove("d-none");
+				} else {
+					document.getElementById(key).classList.add("d-none");
+				}
+			});
 		}
 
-		hint = document.createElement("div");
-		hint.classList.add("alert");
-		hint.classList.add("alert-warning");
-		hint.classList.add("my-1");
-		hint.innerHTML = text;
-
-		ele.parentNode.appendChild(hint);
-	}
-
-	let hint_mtmods = `Tip:
-		Don't include <i>Minetest</i>, <i>mod</i>, or <i>modpack</i> anywhere in the short description.
-		It is unnecessary and wastes characters.`;
-
-	let hint_thegame = `Tip:
-		It's obvious that this adds something to Minetest,
-		there's no need to use phrases such as \"adds X to the game\".`;
-
-	const shortDescField = document.getElementById("short_desc");
-
-	function handleShortDescChange() {
-		const val = shortDescField.value.toLowerCase();
-		if (val.indexOf("minetest") >= 0 || val.indexOf("mod") >= 0 ||
-				val.indexOf("modpack") >= 0 || val.indexOf("mod pack") >= 0) {
-			showHint(shortDescField, hint_mtmods);
-		} else if (val.indexOf("the game") >= 0) {
-			showHint(shortDescField, hint_thegame);
-		} else if (hint) {
-			hint.remove();
-			hint = null;
+		const field = document.getElementById(id);
+		if (field.easy_mde) {
+			field.easy_mde.codemirror.on("change", () => {
+				const value = field.easy_mde.value();
+				onChange(value);
+			});
+		} else {
+			field.addEventListener("change", () => onChange(field.value));
+			field.addEventListener("paste", () => onChange(field.value));
+			field.addEventListener("keyup", () => onChange(field.value));
+			field.addEventListener("input", () => onChange(field.value));
 		}
+		onChange(field.value);
 	}
 
-	shortDescField.addEventListener("change", handleShortDescChange);
-	shortDescField.addEventListener("paste", handleShortDescChange);
-	shortDescField.addEventListener("keyup", handleShortDescChange);
+	setupHints("short_desc", {
+		"short_desc_mods": (val) => val.indexOf("minetest") >= 0 || val.indexOf("mod") >= 0 ||
+				val.indexOf("modpack") >= 0 || val.indexOf("mod pack") >= 0,
+	});
+
+	setupHints("desc", {
+		"desc_page_link": (val) => {
+			let packageUrl = window.location.href.replace("/edit/", "");
+			if (packageUrl.indexOf("/packages/new/") >= 0) {
+				const author = document.querySelector("form[data-author]").getAttribute("data-author");
+				const name = document.getElementById("name").value;
+				packageUrl = `/packages/${author}/${name}/`;
+			}
+			return val.indexOf(packageUrl.toLowerCase()) >= 0;
+		},
+		"desc_page_topic": (val) => {
+			const topicId = document.getElementById("forums").value;
+			const r = new RegExp(`forum\\.minetest\\.net\\/viewtopic\\.php\\?[a-z0-9=&]*t=${topicId}`);
+			return topicId && r.test(val);
+		},
+		"desc_page_repo": (val) => {
+			const repoUrl = document.getElementById("repo").value.replace(".git", "");
+			return repoUrl && val.indexOf(repoUrl.toLowerCase()) >= 0;
+		},
+	});
 })

app/public/static/js/release_minmax.js

@@ -7,15 +7,23 @@ window.addEventListener("load", () => {
 	const min = document.getElementById("min_rel");
 	const max = document.getElementById("max_rel");
 	const none = parseInt(document.querySelector("#min_rel option:first-child").value);
-	const warning = document.getElementById("minmax_warning");
+	const latestMax = parseInt(document.querySelector("#max_rel option:last-child").value);
+	const warningMinMax = document.getElementById("minmax_warning");
+	const warningMax = document.getElementById("latest_release");
 
 	function ver_check() {
 		const minv = parseInt(min.value);
 		const maxv = parseInt(max.value);
 		if (minv != none && maxv != none && minv > maxv) {
-			warning.style.display = "block";
+			warningMinMax.classList.remove("d-none");
 		} else {
-			warning.style.display = "none";
+			warningMinMax.classList.add("d-none");
+		}
+
+		if (maxv == latestMax) {
+			warningMax.classList.remove("d-none");
+		} else {
+			warningMax.classList.add("d-none");
 		}
 	}
 

app/public/static/js/release_new.js

@@ -5,15 +5,15 @@
 
 window.addEventListener("load", () => {
 	function check_opt() {
-		if (document.querySelector("input[name='uploadOpt']:checked").value === "vcs") {
+		if (document.querySelector("input[name='upload_mode']:checked").value === "vcs") {
 			document.getElementById("file_upload").parentElement.classList.add("d-none");
-			document.getElementById("vcsLabel").parentElement.classList.remove("d-none");
+			document.getElementById("vcs_label").parentElement.classList.remove("d-none");
 		} else {
 			document.getElementById("file_upload").parentElement.classList.remove("d-none");
-			document.getElementById("vcsLabel").parentElement.classList.add("d-none");
+			document.getElementById("vcs_label").parentElement.classList.add("d-none");
 		}
 	}
 
-	document.querySelectorAll("input[name='uploadOpt']").forEach(x => x.addEventListener("change", check_opt));
+	document.querySelectorAll("input[name='upload_mode']").forEach(x => x.addEventListener("change", check_opt));
 	check_opt();
 });

app/public/static/js/tagselector.js

@@ -53,7 +53,7 @@
 					text = text.substr(0, idx);
 				}
 
-				$('<span class="badge roaded-pill bg-primary"/>')
+				$('<span class="badge rounded-pill bg-primary"/>')
 					.text(text + ' ')
 					.data("id", id)
 					.append('<a>x</a>')

app/public/static/js/topic_discard.js

@@ -1,33 +0,0 @@
-// @author rubenwardy
-// @license magnet:?xt=urn:btih:1f739d935676111cfff4b4693e3816e664797050&dn=gpl-3.0.txt GPL-v3-or-Later
-
-"use strict";
-
-document.querySelectorAll(".topic-discard").forEach(ele => ele.addEventListener("click", (e) => {
-	const row = ele.parentNode.parentNode;
-	const tid = ele.getAttribute("data-tid");
-	const discard = !row.classList.contains("discardtopic");
-	fetch(new Request("/api/topic_discard/?tid=" + tid +
-			"&discard=" + (discard ? "true" : "false"), {
-		method: "post",
-		credentials: "same-origin",
-		headers: {
-			"Accept": "application/json",
-			"X-CSRFToken": csrf_token,
-		},
-	})).then(function(response) {
-		response.text().then(function(txt) {
-			if (JSON.parse(txt).discarded) {
-				row.classList.add("discardtopic");
-				ele.classList.remove("btn-danger");
-				ele.classList.add("btn-success");
-				ele.innerText = "Show";
-			} else {
-				row.classList.remove("discardtopic");
-				ele.classList.remove("btn-success");
-				ele.classList.add("btn-danger");
-				ele.innerText = "Discard";
-			}
-		}).catch(console.error);
-	}).catch(console.error);
-}));

app/querybuilder.py

@@ -14,9 +14,10 @@
 # You should have received a copy of the GNU Affero General Public License
 # along with this program.  If not, see <https://www.gnu.org/licenses/>.
 
-from flask import abort, current_app
-from flask_babel import lazy_gettext
-from sqlalchemy import or_
+from typing import Optional, List
+from flask import abort, current_app, request, make_response
+from flask_babel import lazy_gettext, gettext, get_locale
+from sqlalchemy import or_, and_
 from sqlalchemy.orm import subqueryload
 from sqlalchemy.sql.expression import func
 from sqlalchemy_searchable import search
@@ -27,8 +28,29 @@ from .utils import is_yes, get_int_or_abort
 
 
 class QueryBuilder:
-	types  = None
-	search = None
+	emit_http_errors: bool
+	limit: Optional[int]
+	lang: str = "en"
+	types: List[PackageType]
+	search: Optional[str] = None
+	only_approved: bool = True
+	licenses: List[License]
+	tags: List[Tag]
+	hide_tags: List[Tag]
+	game: Optional[Package]
+	author: Optional[str]
+	random: bool
+	lucky: bool
+	order_dir: str
+	order_by: Optional[str]
+	flags: set[str]
+	hide_flags: set[str]
+	hide_deprecated: bool
+	hide_wip: bool
+	hide_nonfree: bool
+	show_added: bool
+	version: Optional[MinetestRelease]
+	has_lang: Optional[str]
 
 	@property
 	def title(self):
@@ -40,36 +62,76 @@ class QueryBuilder:
 		if len(self.tags) == 0:
 			ret = package_type
 		elif len(self.tags) == 1:
-			ret = self.tags[0].title + " " + package_type
+			ret = self.tags[0].get_translated()["title"] + " " + package_type
 		else:
-			tags = ", ".join([tag.title for tag in self.tags])
+			tags = ", ".join([tag.get_translated()["title"] for tag in self.tags])
 			ret = f"{tags} - {package_type}"
 
 		if self.search:
 			ret = f"{self.search} - {ret}"
 
+		if self.game:
+			meta = self.game.get_translated(load_desc=False)
+			ret = gettext("%(package_type)s for %(game_name)s", package_type=ret, game_name=meta["title"])
+
 		return ret
 
 	@property
-	def noindex(self):
-		return (self.search is not None or len(self.tags) > 1 or len(self.types) > 1 or len(self.hide_flags) > 0 or
-			self.random or self.lucky or self.author or self.version or self.game)
+	def query_hint(self):
+		return self.title
 
-	def __init__(self, args):
+	@property
+	def noindex(self):
+		return (self.search is not None or len(self.tags) > 1 or len(self.flags) > 1 or len(self.types) > 1 or
+			len(self.licenses) > 0 or len(self.hide_flags) > 0 or len(self.hide_tags) > 0 or self.random or
+			self.lucky or self.author or self.version or self.game or self.limit is not None)
+
+	def __init__(self, args, cookies: bool = False, lang: Optional[str] = None, emit_http_errors: bool = True):
+		self.emit_http_errors = emit_http_errors
+
+		if lang is None:
+			locale = get_locale()
+			if locale:
+				self.lang = locale.language
+		else:
+			self.lang = lang
 
 		# Get request types
 		types = args.getlist("type")
 		types = [PackageType.get(tname) for tname in types]
-		types = [type for type in types if type is not None]
+		if not emit_http_errors:
+			types = [type for type in types if type is not None]
+		elif any([type is None for type in types]):
+			abort(make_response("Unknown type"), 400)
 
 		# Get tags types
 		tags = args.getlist("tag")
 		tags = [Tag.query.filter_by(name=tname).first() for tname in tags]
-		tags = [tag for tag in tags if tag is not None]
+		if not emit_http_errors:
+			tags = [tag for tag in tags if tag is not None]
+		elif any([tag is None for tag in tags]):
+			abort(make_response("Unknown tag"), 400)
 
 		# Hide
 		self.hide_flags = set(args.getlist("hide"))
 
+		self.hide_tags = []
+		for flag in set(self.hide_flags):
+			tag = Tag.query.filter_by(name=flag).first()
+			if tag is not None:
+				self.hide_tags.append(tag)
+				self.hide_flags.remove(flag)
+
+		# Show flags
+		self.flags = set(args.getlist("flag"))
+
+		# License
+		self.licenses = [License.query.filter(func.lower(License.name) == name.lower()).first() for name in args.getlist("license")]
+		if emit_http_errors and any(map(lambda x: x is None, self.licenses)):
+			all_licenses = db.session.query(License.name).order_by(db.asc(License.name)).all()
+			all_licenses = [x[0] for x in all_licenses]
+			abort(make_response("Unknown license. Expected license name from: " + ", ".join(all_licenses)), 400)
+
 		self.types  = types
 		self.tags   = tags
 
@@ -77,6 +139,8 @@ class QueryBuilder:
 		self.lucky  = "lucky" in args
 		self.limit  = 1 if self.lucky else get_int_or_abort(args.get("limit"), None)
 		self.order_by  = args.get("sort")
+		if self.order_by == "":
+			self.order_by = None
 		self.order_dir = args.get("order") or "desc"
 
 		if "android_default" in self.hide_flags:
@@ -100,12 +164,14 @@ class QueryBuilder:
 
 		protocol_version = get_int_or_abort(args.get("protocol_version"))
 		minetest_version = args.get("engine_version")
+		if minetest_version == "":
+			minetest_version = None
+
 		if protocol_version or minetest_version:
 			self.version = MinetestRelease.get(minetest_version, protocol_version)
 		else:
 			self.version = None
 
-		self.show_discarded = is_yes(args.get("show_discarded"))
 		self.show_added = args.get("show_added")
 		if self.show_added is not None:
 			self.show_added = is_yes(self.show_added)
@@ -116,6 +182,17 @@ class QueryBuilder:
 		self.game = args.get("game")
 		if self.game:
 			self.game = Package.get_by_key(self.game)
+			if self.game is None:
+				abort(make_response("Unable to find that game"), 400)
+		else:
+			self.game = None
+
+		self.has_lang = args.get("lang")
+		if self.has_lang == "":
+			self.has_lang = None
+
+		if cookies and request.cookies.get("hide_nonfree") == "1":
+			self.hide_nonfree = True
 
 	def set_sort_if_none(self, name, dir="desc"):
 		if self.order_by is None:
@@ -136,23 +213,26 @@ class QueryBuilder:
 
 		return releases_query.all()
 
-	def convert_to_dictionary(self, packages):
+	def convert_to_dictionary(self, packages, include_vcs: bool):
 		releases = {}
 		for [package_id, release_id] in self.get_releases():
 			releases[package_id] = release_id
 
 		def to_json(package: Package):
 			release_id = releases.get(package.id)
-			return package.as_short_dict(current_app.config["BASE_URL"], release_id=release_id, no_load=True)
+			return package.as_short_dict(current_app.config["BASE_URL"], release_id=release_id, no_load=True,
+					lang=self.lang, include_vcs=include_vcs)
 
 		return [to_json(pkg) for pkg in packages]
 
 	def build_package_query(self):
 		if self.order_by == "last_release":
-			query = db.session.query(Package).select_from(PackageRelease).join(Package) \
-				.filter_by(state=PackageState.APPROVED)
+			query = db.session.query(Package).select_from(PackageRelease).join(Package)
 		else:
-			query = Package.query.filter_by(state=PackageState.APPROVED)
+			query = Package.query
+
+		if self.only_approved:
+			query = query.filter(Package.state == PackageState.APPROVED)
 
 		query = query.options(subqueryload(Package.main_screenshot), subqueryload(Package.aliases))
 
@@ -177,8 +257,14 @@ class QueryBuilder:
 		if self.game:
 			query = query.filter(Package.supported_games.any(game=self.game, supports=True))
 
+		if self.has_lang and self.has_lang != "en":
+			query = query.filter(Package.translations.any(language_id=self.has_lang))
+
 		for tag in self.tags:
-			query = query.filter(Package.tags.any(Tag.id == tag.id))
+			query = query.filter(Package.tags.contains(tag))
+
+		for tag in self.hide_tags:
+			query = query.filter(~Package.tags.contains(tag))
 
 		if "*" in self.hide_flags:
 			query = query.filter(~ Package.content_warnings.any())
@@ -187,6 +273,33 @@ class QueryBuilder:
 				warning = ContentWarning.query.filter_by(name=flag).first()
 				if warning:
 					query = query.filter(~ Package.content_warnings.any(ContentWarning.id == warning.id))
+				elif self.emit_http_errors:
+					abort(make_response("Unknown tag or content warning " + flag), 400)
+
+		flags = set(self.flags)
+		if "nonfree" in flags:
+			query = query.filter(or_(Package.license.has(is_foss=False), Package.media_license.has(is_foss=False)))
+			flags.discard("nonfree")
+		if "wip" in flags:
+			query = query.filter(Package.dev_state == PackageDevState.WIP)
+			flags.discard("wip")
+		if "deprecated" in flags:
+			query = query.filter(Package.dev_state == PackageDevState.DEPRECATED)
+			flags.discard("deprecated")
+
+		if "*" in flags:
+			query = query.filter(Package.content_warnings.any())
+			flags.discard("*")
+		else:
+			for flag in flags:
+				warning = ContentWarning.query.filter_by(name=flag).first()
+				if warning:
+					query = query.filter(Package.content_warnings.any(ContentWarning.id == warning.id))
+
+		licenses = [Package.license_id == license.id for license in self.licenses if license is not None]
+		licenses.extend([Package.media_license_id == license.id for license in self.licenses if license is not None])
+		if len(licenses) > 0:
+			query = query.filter(or_(*licenses))
 
 		if self.hide_nonfree:
 			query = query.filter(Package.license.has(License.is_foss == True))
@@ -198,12 +311,9 @@ class QueryBuilder:
 			query = query.filter(or_(Package.dev_state==None, Package.dev_state != PackageDevState.DEPRECATED))
 
 		if self.version:
-			query = query.join(Package.releases) \
-				.filter(PackageRelease.approved == True) \
-				.filter(or_(PackageRelease.min_rel_id==None,
-					PackageRelease.min_rel_id <= self.version.id)) \
-				.filter(or_(PackageRelease.max_rel_id==None,
-					PackageRelease.max_rel_id >= self.version.id))
+			query = query.filter(Package.releases.any(and_(or_(PackageRelease.min_rel_id==None,
+					PackageRelease.min_rel_id <= self.version.id), or_(PackageRelease.max_rel_id==None,
+					PackageRelease.max_rel_id >= self.version.id))))
 
 		return query
 
@@ -234,7 +344,7 @@ class QueryBuilder:
 		elif self.order_by == "approved_at" or self.order_by == "date":
 			to_order = Package.approved_at
 		elif self.order_by == "last_release":
-			to_order = PackageRelease.releaseDate
+			to_order = PackageRelease.created_at
 		else:
 			abort(400)
 
@@ -253,9 +363,6 @@ class QueryBuilder:
 	def build_topic_query(self, show_added=False):
 		query = ForumTopic.query
 
-		if not self.show_discarded:
-			query = query.filter_by(discarded=False)
-
 		show_added = self.show_added == True or (self.show_added is None and show_added)
 		if not show_added:
 			query = query.filter(~ db.exists().where(Package.forums==ForumTopic.topic_id))

app/rediscache.py

@@ -20,12 +20,15 @@ from . import redis_client
 # and also means that the rest of the code avoids knowing about `app`
 
 
+EXPIRY_TIME_S = 2*7*24*60*60  # 2 weeks
+
+
 def make_download_key(ip, package):
 	return "{}/{}/{}".format(ip, package.author.username, package.name)
 
 
-def set_key(key, v):
-	redis_client.set(key, v)
+def set_temp_key(key, v):
+	redis_client.set(key, v, ex=EXPIRY_TIME_S)
 
 
 def has_key(key):