typo 2

Fixes #598

app/__init__.py

@@ -21,13 +21,12 @@ import redis
 from flask import redirect, url_for, render_template, flash, request, Flask, send_from_directory, make_response, render_template_string
 from flask_babel import Babel, gettext
 from flask_flatpages import FlatPages
-from flask_flatpages.utils import pygmented_markdown
 from flask_github import GitHub
 from flask_login import logout_user, current_user, LoginManager
 from flask_mail import Mail
 from flask_wtf.csrf import CSRFProtect
 
-from app.markdown import init_markdown, MARKDOWN_EXTENSIONS, MARKDOWN_EXTENSION_CONFIG
+from app.markdown import init_markdown, render_markdown
 
 import sentry_sdk
 from sentry_sdk.integrations.flask import FlaskIntegration
@@ -67,13 +66,11 @@ app = Flask(__name__, static_folder="public/static")
 def my_flatpage_renderer(text):
 	# Render with jinja first
 	prerendered_body = render_template_string(text)
-	return pygmented_markdown(prerendered_body, flatpages=pages)
+	return render_markdown(prerendered_body, clean=False)
 
 
 app.config["FLATPAGES_ROOT"] = "flatpages"
 app.config["FLATPAGES_EXTENSION"] = ".md"
-app.config["FLATPAGES_MARKDOWN_EXTENSIONS"] = MARKDOWN_EXTENSIONS
-app.config["FLATPAGES_EXTENSION_CONFIG"] = MARKDOWN_EXTENSION_CONFIG
 app.config["FLATPAGES_HTML_RENDERER"] = my_flatpage_renderer
 app.config["WTF_CSRF_TIME_LIMIT"] = None
 

app/blueprints/threads/__init__.py

@@ -29,7 +29,7 @@ from app.models import Package, db, User, Permission, Thread, UserRank, AuditSev
 from app.utils import add_notification, is_yes, add_audit_log, get_system_user, has_blocked_domains, \
 	normalize_line_endings
 from flask_wtf import FlaskForm
-from wtforms import StringField, TextAreaField, SubmitField, BooleanField
+from wtforms import StringField, TextAreaField, SubmitField
 from wtforms.validators import InputRequired, Length
 from app.utils import get_int_or_abort
 

app/blueprints/zipgrep/__init__.py

@@ -14,27 +14,27 @@
 # You should have received a copy of the GNU Affero General Public License
 # along with this program.  If not, see <https://www.gnu.org/licenses/>.
 
-
 from celery import uuid
 from flask import Blueprint, render_template, redirect, request, abort, url_for
 from flask_babel import lazy_gettext
 from flask_wtf import FlaskForm
-from wtforms import StringField, BooleanField, SubmitField
-from wtforms.validators import InputRequired, Length
+from wtforms import StringField, BooleanField, SubmitField, SelectMultipleField
+from wtforms.validators import InputRequired, Length, Optional
 
 from app.tasks import celery
 from app.utils import rank_required
 
 bp = Blueprint("zipgrep", __name__)
 
-from app.models import UserRank, Package
+from app.models import UserRank, Package, PackageType
 from app.tasks.zipgrep import search_in_releases
 
 
 class SearchForm(FlaskForm):
 	query = StringField(lazy_gettext("Text to find (regex)"), [InputRequired(), Length(1, 100)])
 	file_filter = StringField(lazy_gettext("File filter"), [InputRequired(), Length(1, 100)], default="*.lua")
-	remember_me = BooleanField(lazy_gettext("Remember me"), default=True)
+	type = SelectMultipleField(lazy_gettext("Type"), [Optional()],
+			choices=PackageType.choices(), coerce=PackageType.coerce)
 	submit = SubmitField(lazy_gettext("Search"))
 
 
@@ -44,7 +44,7 @@ def zipgrep_search():
 	form = SearchForm(request.form)
 	if form.validate_on_submit():
 		task_id = uuid()
-		search_in_releases.apply_async((form.query.data, form.file_filter.data), task_id=task_id)
+		search_in_releases.apply_async((form.query.data, form.file_filter.data, [x.name for x in form.type.data]), task_id=task_id)
 		result_url = url_for("zipgrep.view_results", id=task_id)
 		return redirect(url_for("tasks.check", id=task_id, r=result_url))
 

app/flatpages/help/wtfpl.md

@@ -1,25 +1,6 @@
 title: WTFPL is a terrible license
 toc: False
 
-<div id="warning" class="alert alert-warning">
-	<span class="icon_message"></span>
-
-	Please reconsider the choice of WTFPL as a license.
-
-	<script>
-		// @author rubenwardy
-		// @license magnet:?xt=urn:btih:1f739d935676111cfff4b4693e3816e664797050&dn=gpl-3.0.txt GPL-v3-or-Later
-
-		var params = new URLSearchParams(location.search);
-		var r = params.get("r");
-		if (r) {
-			document.write("<a class='alert_right button' href='" + r + "'>Okay</a>");
-		} else {
-			document.getElementById("warning").style.display = "none";
-        }
-	</script>
-</div>
-
 The use of WTFPL as a license is discouraged for multiple reasons.
 
 * **No Warranty disclaimer:** This could open you up to being sued.<sup>[1]</sup>
@@ -37,4 +18,4 @@ license, saying:<sup>[3]</sup>
 
 1. [WTFPL is harmful to software developers](https://cubicspot.blogspot.com/2017/04/wtfpl-is-harmful-to-software-developers.html)
 2. [FSF](https://www.gnu.org/licenses/license-list.en.html)
-3. [OSI](https://opensource.org/minutes20090304)
+3. [OSI](https://opensource.org/meeting-minutes/minutes20090304)

app/flatpages/policy_and_guidance.md

@@ -1,22 +1,5 @@
 title: Package Inclusion Policy and Guidance
 
-## 0. Overview
-
-ContentDB is for the community, and as such listings should be useful to the
-community. To help with this, there are a few rules to improve the quality of
-the listings and to combat abuse.
-
-* **No inappropriate content.** <sup>2.1</sup>
-* **Content must be playable/useful, but not necessarily finished.** <sup>2.2</sup>
-* **Don't use the name of another mod unless your mod is a fork or reimplementation.** <sup>3</sup>
-* **Licenses must allow derivatives, redistribution, and must not discriminate.** <sup>4</sup>
-* **Don't put promotions or advertisements in any package metadata.** <sup>5</sup>
-* **Don't manipulate package placement using reviews or downloads.** <sup>6</sup>
-* **Screenshots must not be misleading.** <sup>7</sup>
-* **The ContentDB admin reserves the right to remove packages for any reason**,
-  including ones not covered by this document, and to ban users who abuse
-  this service. <sup>1</sup>
-
 
 ## 1. General
 
@@ -26,35 +9,53 @@ including ones not covered by this document, and to ban users who abuse this ser
 
 ## 2. Accepted Content
 
-### 2.1. Acceptable Content
+### 2.1. Mature Content
 
-Sexually-orientated content is not permitted.
-If in doubt at what this means, [contact us by raising a report](/report/).
+See the [Terms of Service](/terms/) for a full list of prohibited content.
 
-Content which depicts or encourages the use of illegal drugs (under the laws of the United Kingdom) is not permitted.
+Other mature content is permitted providing that it is labelled with the applicable
+[content warning](/help/content_flags/).
 
-Mature content is permitted providing that it is labelled correctly.
-See [Content Flags](/help/content_flags/).
+### 2.2. Useful Content / State of Completion
 
-### 2.2. State of Completion
+ContentDB is for playable and useful content - content which is sufficiently
+complete to be useful to end-users.
 
-ContentDB should only currently contain playable content - content which is
-sufficiently complete to be useful to end-users. It's fine to add stuff which is
-still a Work in Progress (WIP) as long as it adds sufficient value; Note that
-this doesn't mean that you should add a thing you started working on yesterday,
-it's worth adding all the basic stuff to make your package useful.
-
-You should make sure to mark Work in Progress stuff as such in the "maintenance
-status" column, as this will help advise players.
+It's fine to add stuff which is still a Work in Progress (WIP) as long as it
+adds sufficient value. You must make sure to mark Work in Progress stuff as
+such in the "maintenance status" dropdown, as this will help advise players.
 
 Adding non-player facing mods, such as libraries and server tools, is perfectly
-fine and encouraged. ContentDB isn't just for player-facing things, and adding
-libraries allows them to be installed when a mod depends on it.
+fine and encouraged. ContentDB isn't just for player-facing things and adding
+libraries allows Luanti to automatically install dependencies.
+
+### 2.3. Language
+
+We require packages to be in English with (optional) client-side translations for
+other languages. This is because Luanti currently requires English as the base language
+([Issue to change this](https://github.com/luanti-org/luanti/issues/6503)).
+
+Your package's title and short description must be in English. You can use client-side
+translations to [translate content meta](https://api.luanti.org/translations/#translating-content-meta).
+
+### 2.4. Attempt to contribute before forking
+
+You should attempt to contribute upstream before forking a package. If you choose
+to fork, you should have a justification (different objectives, maintainer is unavailable, etc).
+You should use a different title and make it clear in the long description what the
+benefit of your fork is over the original package.
+
+### 2.5. Copyright and trademarks
+
+Your package must not violate copyright or trademarks. You should avoid the use of
+trademarks in the package's title or short description. If you do use a trademark,
+ensure that you phrase it in a way that does not imply official association or
+endorsement.
 
 
 ## 3. Technical Names
 
-### 3.1 Right to a name
+### 3.1. Right to a Name
 
 A package uses a name when it has that name or contains a mod that uses that name.
 
@@ -72,23 +73,46 @@ to change the name of the package, or your package won't be accepted.
 
 We reserve the right to issue exceptions for this where we feel necessary.
 
-### 3.2. Mod Forks and Reimplementations
+### 3.2. Forks and Reimplementations
 
 An exception to the above is that mods are allowed to have the same name as a
 mod if it's a fork of that mod (or a close reimplementation). In real terms, it
-should be possible to use the new mod as a drop-in replacement.
+must be possible to use the new mod as a drop-in replacement.
 
 We reserve the right to decide whether a mod counts as a fork or
 reimplementation of the mod that owns the name.
 
+### 3.3. Game Mod Namespacing
+
+New mods introduced by a game must have a unique common prefix to avoid conflicts with
+other games and standalone mods. For example, the NodeCore game's first-party mods all
+start with `nc_`: `nc_api`, `nc_doors`.
+
+You may include existing or standard mods in your game without renaming them to use the
+namespace. For example, NodeCore could include the `awards` mod without needing to rename it.
+
+Standalone mods may not use a game's namespace unless they have been given permission by
+the game's author.
+
+The exception given by 3.2 also applies to game namespaces - you may use another game's
+prefix if your game is a fork.
+
 
 ## 4. Licenses
 
-### 4.1. Allowed Licenses
+### 4.1. License file
 
+You must have a LICENSE, LICENSE.txt, or LICENSE.md file describing the licensing of your package.
 Please ensure that you correctly credit any resources (code, assets, or otherwise)
-that you have used in your package. For help on doing copyright correctly, see
-the [Copyright help page](/help/copyright/).
+that you have used in your package.
+
+You may use lowercase or include a suffix in the filename (ie: `license-code.txt`). If
+you are making a game or modpack, your top level license file may just be a summary or
+refer to the license files of individual components.
+
+For help on doing copyright correctly, see the [Copyright help page](/help/copyright/).
+
+### 4.2. Allowed Licenses
 
 **The use of licenses that do not allow derivatives or redistribution is not
 permitted. This includes CC-ND (No-Derivatives) and lots of closed source licenses.
@@ -98,13 +122,13 @@ of the content on servers or singleplayer is also not permitted.**
 However, closed sourced licenses are allowed if they allow the above.
 
 If the license you use is not on the list then please select "Other", and we'll
-get around to adding it. We tend to reject custom/untested licenses, and
-reserve the right to decide whether a license should be included.
+get around to adding it. We reject custom/untested licenses and reserve the right
+to decide whether a license should be included.
 
 Please note that the definitions of "free" and "non-free" is the same as that
 of the [Free Software Foundation](https://www.gnu.org/philosophy/free-sw.en.html).
 
-### 4.2. Recommended Licenses
+### 4.3. Recommended Licenses
 
 It is highly recommended that you use a Free and Open Source software (FOSS)
 license. FOSS licenses result in a sharing community and will increase the
@@ -152,10 +176,14 @@ Doing so may result in temporary or permanent suspension from ContentDB.
 
 ## 7. Screenshots
 
-1.  **Screenshots must not violate copyright.** You should have the rights to the
-    screenshot.
+1.  We require all packages to have at least one screenshot. For packages without visual
+    content, we recommend making a symbolic image with icons, graphics, or text to depict
+    the package.
 
-2.  **Screenshots must depict the actual content of the package in some way, and
+2.  **Screenshots must not violate copyright.** This means don't just copy images
+    from Google search, see [the copyright guide](/help/copyright/).
+
+3.  **Screenshots must depict the actual content of the package in some way, and
     not be misleading.**
 
     Do not use idealized mockups or blender concept renders if they do not
@@ -171,20 +199,9 @@ Doing so may result in temporary or permanent suspension from ContentDB.
     will look like in a typical/realistic game scene, but should be "in the
     background" only as far as possible.
 
-3.  **Screenshots must only contain content appropriate for the Content Warnings of
+4.  **Screenshots must only contain content appropriate for the Content Warnings of
     the package.**
 
-4.  **Screenshots should be MOSTLY in-game screenshots, if applicable.** Some
-    alterations on in-game screenshots are okay, such as collages, added text,
-    some reasonable compositing.
-
-    Don't just use one of the textures from the package; show it in-situ as it
-    actually looks in the game.
-
-5.  **Packages should have a screenshot when reasonably applicable.**
-
-6.  **Screenshots should be of reasonable dimensions.** We recommend using 1920x1080.
-
 
 ## 8. Security
 

app/markdown.py

@@ -1,214 +0,0 @@
-# ContentDB
-# Copyright (C) rubenwardy
-#
-# This program is free software: you can redistribute it and/or modify
-# it under the terms of the GNU Affero General Public License as published by
-# the Free Software Foundation, either version 3 of the License, or
-# (at your option) any later version.
-#
-# This program is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-# GNU Affero General Public License for more details.
-#
-# You should have received a copy of the GNU Affero General Public License
-# along with this program.  If not, see <https://www.gnu.org/licenses/>.
-
-from functools import partial
-from urllib.parse import urljoin
-
-import bleach
-from bleach import Cleaner
-from bleach.linkifier import LinkifyFilter
-from bs4 import BeautifulSoup
-from markdown import Markdown
-from flask import url_for
-from jinja2.utils import markupsafe
-from markdown.extensions import Extension
-from markdown.inlinepatterns import SimpleTagInlineProcessor
-from markdown.inlinepatterns import Pattern
-from markdown.extensions.codehilite import CodeHiliteExtension
-from xml.etree import ElementTree
-
-# Based on
-# https://github.com/Wenzil/mdx_bleach/blob/master/mdx_bleach/whitelist.py
-#
-# License: MIT
-
-ALLOWED_TAGS = {
-	"h1", "h2", "h3", "h4", "h5", "h6", "hr",
-	"ul", "ol", "li",
-	"p",
-	"br",
-	"pre",
-	"code",
-	"blockquote",
-	"strong",
-	"em",
-	"a",
-	"img",
-	"table", "thead", "tbody", "tr", "th", "td",
-	"div", "span", "del", "s",
-	"details",
-	"summary",
-}
-
-ALLOWED_CSS = [
-	"highlight", "codehilite",
-	"hll", "c", "err", "g", "k", "l", "n", "o", "x", "p", "ch", "cm", "cp", "cpf", "c1", "cs",
-	"gd", "ge", "gr", "gh", "gi", "go", "gp", "gs", "gu", "gt", "kc", "kd", "kn", "kp", "kr",
-	"kt", "ld", "m", "s", "na", "nb", "nc", "no", "nd", "ni", "ne", "nf", "nl", "nn", "nx",
-	"py", "nt", "nv", "ow", "w", "mb", "mf", "mh", "mi", "mo", "sa", "sb", "sc", "dl", "sd",
-	"s2", "se", "sh", "si", "sx", "sr", "s1", "ss", "bp", "fm", "vc", "vg", "vi", "vm", "il",
-]
-
-
-def allow_class(_tag, name, value):
-	return name == "class" and value in ALLOWED_CSS
-
-
-ALLOWED_ATTRIBUTES = {
-	"h1": ["id"],
-	"h2": ["id"],
-	"h3": ["id"],
-	"h4": ["id"],
-	"a": ["href", "title", "data-username"],
-	"img": ["src", "title", "alt"],
-	"code": allow_class,
-	"div": allow_class,
-	"span": allow_class,
-	"table": ["id"],
-}
-
-ALLOWED_PROTOCOLS = {"http", "https", "mailto"}
-
-md = None
-
-
-def linker_callback(attrs, new=False):
-	if new:
-		text = attrs.get("_text")
-		if not (text.startswith("http://") or text.startswith("https://")):
-			return None
-	return attrs
-
-
-def render_markdown(source):
-	html = md.convert(source)
-
-	cleaner = Cleaner(
-		tags=ALLOWED_TAGS,
-		attributes=ALLOWED_ATTRIBUTES,
-		protocols=ALLOWED_PROTOCOLS,
-		filters=[partial(LinkifyFilter,
-				callbacks=[linker_callback] + bleach.linkifier.DEFAULT_CALLBACKS,
-				skip_tags={"pre", "code"})])
-	return cleaner.clean(html)
-
-
-class DelInsExtension(Extension):
-	def extendMarkdown(self, md):
-		del_proc = SimpleTagInlineProcessor(r"(\~\~)(.+?)(\~\~)", "del")
-		md.inlinePatterns.register(del_proc, "del", 200)
-
-		ins_proc = SimpleTagInlineProcessor(r"(\+\+)(.+?)(\+\+)", "ins")
-		md.inlinePatterns.register(ins_proc, "ins", 200)
-
-
-RE_PARTS = dict(
-	USER=r"[A-Za-z0-9._-]*\b",
-	REPO=r"[A-Za-z0-9_]+\b"
-)
-
-
-class MentionPattern(Pattern):
-	ANCESTOR_EXCLUDES = ("a",)
-
-	def __init__(self, config, md):
-		MENTION_RE = r"(@({USER})(?:\/({REPO}))?)".format(**RE_PARTS)
-		super(MentionPattern, self).__init__(MENTION_RE, md)
-		self.config = config
-
-	def handleMatch(self, m):
-		from app.models import User
-
-		label = m.group(2)
-		user = m.group(3)
-		package_name = m.group(4)
-		if package_name:
-			el = ElementTree.Element("a")
-			el.text = label
-			el.set("href", url_for("packages.view", author=user, name=package_name))
-			return el
-		else:
-			if User.query.filter_by(username=user).count() == 0:
-				return None
-
-			el = ElementTree.Element("a")
-			el.text = label
-			el.set("href", url_for("users.profile", username=user))
-			el.set("data-username", user)
-			return el
-
-
-class MentionExtension(Extension):
-	def __init__(self, *args, **kwargs):
-		super(MentionExtension, self).__init__(*args, **kwargs)
-
-	def extendMarkdown(self, md):
-		md.ESCAPED_CHARS.append("@")
-		md.inlinePatterns.register(MentionPattern(self.getConfigs(), md), "mention", 20)
-
-
-MARKDOWN_EXTENSIONS = ["fenced_code", "tables", CodeHiliteExtension(guess_lang=False), "toc", DelInsExtension(), MentionExtension()]
-MARKDOWN_EXTENSION_CONFIG = {
-	"fenced_code": {},
-	"tables": {}
-}
-
-
-def init_markdown(app):
-	global md
-
-	md = Markdown(extensions=MARKDOWN_EXTENSIONS,
-			extension_configs=MARKDOWN_EXTENSION_CONFIG,
-			output_format="html")
-
-	@app.template_filter()
-	def markdown(source):
-		return markupsafe.Markup(render_markdown(source))
-
-
-def get_headings(html: str):
-	soup = BeautifulSoup(html, "html.parser")
-	headings = soup.find_all(["h1", "h2", "h3"])
-
-	root = []
-	stack = []
-	for heading in headings:
-		this = {"link": heading.get("id") or "", "text": heading.text, "children": []}
-		this_level = int(heading.name[1:]) - 1
-
-		while this_level <= len(stack):
-			stack.pop()
-
-		if len(stack) > 0:
-			stack[-1]["children"].append(this)
-		else:
-			root.append(this)
-
-		stack.append(this)
-
-	return root
-
-
-def get_user_mentions(html: str) -> set:
-	soup = BeautifulSoup(html, "html.parser")
-	links = soup.select("a[data-username]")
-	return set([x.get("data-username") for x in links])
-
-
-def get_links(html: str, url: str) -> set:
-	soup = BeautifulSoup(html, "html.parser")
-	links = soup.select("a[href]")
-	return set([urljoin(url, x.get("href")) for x in links])

app/markdown/__init__.py

@@ -0,0 +1,113 @@
+# ContentDB
+# Copyright (C) rubenwardy
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program.  If not, see <https://www.gnu.org/licenses/>.
+
+from typing import Sequence
+from urllib.parse import urljoin
+from bs4 import BeautifulSoup
+from jinja2.utils import markupsafe
+from markdown_it import MarkdownIt
+from markdown_it.common.utils import unescapeAll, escapeHtml
+from markdown_it.token import Token
+from markdown_it.presets import gfm_like
+from mdit_py_plugins.anchors import anchors_plugin
+from pygments import highlight
+from pygments.lexers import get_lexer_by_name
+from pygments.util import ClassNotFound
+from pygments.formatters.html import HtmlFormatter
+
+from .cleaner import clean_html
+from .mention import init_mention
+
+
+def highlight_code(code, name, attrs):
+	try:
+		lexer = get_lexer_by_name(name)
+	except ClassNotFound:
+		return None
+
+	formatter = HtmlFormatter()
+
+	return highlight(code, lexer, formatter)
+
+
+def render_code(self, tokens: Sequence[Token], idx, options, env):
+	token = tokens[idx]
+	info = unescapeAll(token.info).strip() if token.info else ""
+	langName = info.split(maxsplit=1)[0] if info else ""
+
+	if options.highlight:
+		return options.highlight(
+			token.content, langName, ""
+		) or f"<pre><code>{escapeHtml(token.content)}</code></pre>"
+
+	return f"<pre><code>{escapeHtml(token.content)}</code></pre>"
+
+
+gfm_like.make()
+md = MarkdownIt("gfm-like", {"highlight": highlight_code})
+md.use(anchors_plugin, permalink=True, permalinkSymbol="#", max_level=6)
+md.add_render_rule("fence", render_code)
+init_mention(md)
+
+
+def render_markdown(source, clean=True):
+	html = md.render(source)
+	if clean:
+		return clean_html(html)
+	else:
+		return html
+
+
+def init_markdown(app):
+	@app.template_filter()
+	def markdown(source):
+		return markupsafe.Markup(render_markdown(source))
+
+
+def get_headings(html: str):
+	soup = BeautifulSoup(html, "html.parser")
+	headings = soup.find_all(["h1", "h2", "h3"])
+
+	root = []
+	stack = []
+	for heading in headings:
+		text = heading.find(text=True, recursive=False)
+		this = {"link": heading.get("id") or "", "text": text, "children": []}
+		this_level = int(heading.name[1:]) - 1
+
+		while this_level <= len(stack):
+			stack.pop()
+
+		if len(stack) > 0:
+			stack[-1]["children"].append(this)
+		else:
+			root.append(this)
+
+		stack.append(this)
+
+	return root
+
+
+def get_user_mentions(html: str) -> set:
+	soup = BeautifulSoup(html, "html.parser")
+	links = soup.select("a[data-username]")
+	return set([x.get("data-username") for x in links])
+
+
+def get_links(html: str, url: str) -> set:
+	soup = BeautifulSoup(html, "html.parser")
+	links = soup.select("a[href]")
+	return set([urljoin(url, x.get("href")) for x in links])

app/markdown/cleaner.py

@@ -0,0 +1,97 @@
+# ContentDB
+# Copyright (C) rubenwardy
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program.  If not, see <https://www.gnu.org/licenses/>.
+
+from functools import partial
+from bleach import Cleaner
+from bleach.linkifier import LinkifyFilter, DEFAULT_CALLBACKS
+
+
+# Based on
+# https://github.com/Wenzil/mdx_bleach/blob/master/mdx_bleach/whitelist.py
+#
+# License: MIT
+
+ALLOWED_TAGS = {
+	"h1", "h2", "h3", "h4", "h5", "h6", "hr",
+	"ul", "ol", "li",
+	"p",
+	"br",
+	"pre",
+	"code",
+	"blockquote",
+	"strong",
+	"em",
+	"a",
+	"img",
+	"table", "thead", "tbody", "tr", "th", "td",
+	"div", "span", "del", "s",
+	"details",
+	"summary",
+	"sup",
+}
+
+ALLOWED_CSS = [
+	"highlight", "codehilite",
+	"hll", "c", "err", "g", "k", "l", "n", "o", "x", "p", "ch", "cm", "cp", "cpf", "c1", "cs",
+	"gd", "ge", "gr", "gh", "gi", "go", "gp", "gs", "gu", "gt", "kc", "kd", "kn", "kp", "kr",
+	"kt", "ld", "m", "s", "na", "nb", "nc", "no", "nd", "ni", "ne", "nf", "nl", "nn", "nx",
+	"py", "nt", "nv", "ow", "w", "mb", "mf", "mh", "mi", "mo", "sa", "sb", "sc", "dl", "sd",
+	"s2", "se", "sh", "si", "sx", "sr", "s1", "ss", "bp", "fm", "vc", "vg", "vi", "vm", "il",
+]
+
+
+def allow_class(_tag, name, value):
+	return name == "class" and value in ALLOWED_CSS
+
+
+def allow_a(_tag, name, value):
+	return name in ["href", "title", "data-username"] or (name == "class" and value == "header-anchor")
+
+
+ALLOWED_ATTRIBUTES = {
+	"h1": ["id"],
+	"h2": ["id"],
+	"h3": ["id"],
+	"h4": ["id"],
+	"a": allow_a,
+	"img": ["src", "title", "alt"],
+	"code": allow_class,
+	"div": allow_class,
+	"span": allow_class,
+	"table": ["id"],
+}
+
+ALLOWED_PROTOCOLS = {"http", "https", "mailto"}
+
+
+def linker_callback(attrs, new=False):
+	if new:
+		text = attrs.get("_text")
+		if not (text.startswith("http://") or text.startswith("https://")):
+			return None
+
+	return attrs
+
+
+def clean_html(html: str):
+	cleaner = Cleaner(
+		tags=ALLOWED_TAGS,
+		attributes=ALLOWED_ATTRIBUTES,
+		protocols=ALLOWED_PROTOCOLS,
+		filters=[partial(LinkifyFilter,
+				callbacks=[linker_callback] + DEFAULT_CALLBACKS,
+				skip_tags={"pre", "code"})])
+	return cleaner.clean(html)

app/markdown/mention.py

@@ -0,0 +1,109 @@
+# ContentDB
+# Copyright (C) rubenwardy
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program.  If not, see <https://www.gnu.org/licenses/>.
+
+import re
+
+from flask import url_for
+from markdown_it import MarkdownIt
+from markdown_it.token import Token
+from markdown_it.rules_core.state_core import StateCore
+from typing import Sequence, List
+
+
+def render_user_mention(self, tokens: Sequence[Token], idx, options, env):
+	token = tokens[idx]
+	username = token.content
+	url = url_for("users.profile", username=username)
+	return f"<a href=\"{url}\" data-username=\"{username}\">@{username}</a>"
+
+
+def render_package_mention(self, tokens: Sequence[Token], idx, options, env):
+	token = tokens[idx]
+	username = token.content
+	name = token.attrs["name"]
+	url = url_for("packages.view", author=username, name=name)
+	return f"<a href=\"{url}\">@{username}/{name}</a>"
+
+
+def parse_mentions(state: StateCore):
+	for block_token in state.tokens:
+		if block_token.type != "inline" or block_token.children is None:
+			continue
+
+		link_depth = 0
+		html_link_depth = 0
+
+		children = []
+		for token in block_token.children:
+			if token.type == "link_open":
+				link_depth += 1
+			elif token.type == "link_close":
+				link_depth -= 1
+			elif token.type == "html_inline":
+				# is link open / close?
+				pass
+
+			if link_depth > 0 or html_link_depth > 0 or token.type != "text":
+				children.append(token)
+			else:
+				children.extend(split_tokens(token, state))
+
+		block_token.children = children
+
+
+RE_PARTS = dict(
+	USER=r"[A-Za-z0-9._-]*\b",
+	NAME=r"[A-Za-z0-9_]+\b"
+)
+MENTION_RE = r"(@({USER})(?:\/({NAME}))?)".format(**RE_PARTS)
+
+
+def split_tokens(token: Token, state: StateCore) -> List[Token]:
+	tokens = []
+	content = token.content
+	pos = 0
+	for match in re.finditer(MENTION_RE, content):
+		username = match.group(2)
+		package_name = match.group(3)
+		(start, end) = match.span(0)
+
+		if start > pos:
+			token_text = Token("text", "", 0)
+			token_text.content = content[pos:start]
+			token_text.level = token.level
+			tokens.append(token_text)
+
+		mention = Token("package_mention" if package_name else "user_mention", "", 0)
+		mention.content = username
+		mention.attrSet("name", package_name)
+		mention.level = token.level
+		tokens.append(mention)
+
+		pos = end
+
+	if pos < len(content):
+		token_text = Token("text", "", 0)
+		token_text.content = content[pos:]
+		token_text.level = token.level
+		tokens.append(token_text)
+
+	return tokens
+
+
+def init_mention(md: MarkdownIt):
+	md.add_render_rule("user_mention", render_user_mention, "html")
+	md.add_render_rule("package_mention", render_package_mention, "html")
+	md.core.ruler.after("inline", "mention", parse_mentions)

app/public/static/js/polltask.js

@@ -22,7 +22,7 @@ function sleep(interval) {
 }
 
 
-async function pollTask(poll_url, disableTimeout) {
+async function pollTask(poll_url, disableTimeout, onProgress) {
 	let tries = 0;
 
 	while (true) {
@@ -42,6 +42,10 @@ async function pollTask(poll_url, disableTimeout) {
 			console.error(e);
 		}
 
+		if (res && res.status) {
+			onProgress?.(res);
+		}
+
 		if (res && res.status === "SUCCESS") {
 			console.log("Got result")
 			return res.result;
@@ -62,3 +66,38 @@ async function performTask(url) {
 		throw "Start task didn't return string!";
 	}
 }
+
+window.addEventListener("load", () => {
+	const taskId = document.querySelector("[data-task-id]")?.getAttribute("data-task-id");
+	if (taskId) {
+		const progress = document.getElementById("progress");
+
+		function onProgress(res) {
+			let status = res.status.toLowerCase();
+			if (status === "progress") {
+				progress.classList.remove("d-none");
+				const bar = progress.children[0];
+
+				const {current, total, running} = res.result;
+				const perc = Math.min(Math.max(100 * current / total, 0), 100);
+				bar.style.width = `${perc}%`;
+				bar.setAttribute("aria-valuenow", current);
+				bar.setAttribute("aria-valuemax", total);
+
+				const packages = running.map(x => `${x.author}/${x.name}`).join(", ");
+				document.getElementById("status").textContent = `Status: in progress (${current} / ${total})\n\n${packages}`;
+			} else {
+				progress.classList.add("d-none");
+
+				if (status === "pending") {
+					status = "pending or unknown";
+				}
+				document.getElementById("status").textContent = `Status: ${status}`;
+			}
+		}
+
+		pollTask(`/tasks/${taskId}/`, true, onProgress)
+			.then(function() { location.reload() })
+			.catch(function() { location.reload() })
+	}
+});

app/scss/custom.scss

@@ -51,6 +51,19 @@ h3 {
 	letter-spacing: .05em
 }
 
+h1, h2, h3, h4, h5, h6 {
+	&::after {
+		display: block;
+		content: "";
+		clear: both;
+	}
+}
+
+.header-anchor {
+	float: right;
+	opacity: 0.8;
+}
+
 .badge-notify {
 	background:yellow; /* #00bc8c;*/
 	color: black;

app/tasks/__init__.py

@@ -64,7 +64,7 @@ class FlaskCelery(Celery):
 
 def make_celery(app):
 	celery = FlaskCelery(app.import_name, backend=app.config['CELERY_RESULT_BACKEND'],
-					broker=app.config['CELERY_BROKER_URL'])
+					broker=app.config['CELERY_BROKER_URL'], task_track_started=True)
 
 	celery.init_app(app)
 	return celery

app/tasks/minetestcheck/tree.py

@@ -25,7 +25,7 @@ from .config import parse_conf
 from .translation import Translation, parse_tr
 
 basenamePattern = re.compile("^([a-z0-9_]+)$")
-licensePattern = re.compile("^(licen[sc]e|copying)(.[^/\n]+)?$", re.IGNORECASE)
+licensePattern = re.compile("^licen[sc]e[^/.]*(\.(txt|md))?$", re.IGNORECASE)
 
 DISALLOWED_NAMES = {
 	"core", "minetest", "group", "table", "string", "lua", "luajit", "assert", "debug",

app/tasks/zipgrep.py

@@ -15,45 +15,70 @@
 # along with this program.  If not, see <https://www.gnu.org/licenses/>.
 
 import subprocess
-from subprocess import Popen, PIPE
-from typing import Optional
+import sys
+from subprocess import Popen, PIPE, TimeoutExpired
+from typing import Optional, List
 
 from app.models import Package, PackageState, PackageRelease
 from app.tasks import celery
 
 
-@celery.task()
-def search_in_releases(query: str, file_filter: str):
-	packages = list(Package.query.filter(Package.state == PackageState.APPROVED).all())
-	running = []
+@celery.task(bind=True)
+def search_in_releases(self, query: str, file_filter: str, types: List[str]):
+	pkg_query = Package.query.filter(Package.state == PackageState.APPROVED)
+	if len(types) > 0:
+		pkg_query = pkg_query.filter(Package.type.in_(types))
+
+	packages = list(pkg_query.all())
 	results = []
 
-	while len(packages) > 0 or len(running) > 0:
-		# Check running
-		for i in range(len(running) - 1, -1, -1):
-			package: Package = running[i][0]
-			handle: subprocess.Popen[str] = running[i][1]
+	total = len(packages)
+	self.update_state(state="PROGRESS", meta={"current": 0, "total": total})
+
+	while len(packages) > 0:
+		package = packages.pop()
+		release: Optional[PackageRelease] = package.get_download_release()
+		if release:
+			print(f"[Zipgrep] Checking {package.name}", file=sys.stderr)
+			self.update_state(state="PROGRESS", meta={
+				"current": total - len(packages),
+				"total": total,
+				"running": [package.as_key_dict()],
+			})
+
+			handle = Popen(["zipgrep", query, release.file_path, file_filter], stdout=PIPE, encoding="UTF-8")
+
+			try:
+				handle.wait(timeout=15)
+			except TimeoutExpired:
+				print(f"[Zipgrep] Timeout for {package.name}", file=sys.stderr)
+				handle.kill()
+				results.append({
+					"package": package.as_key_dict(),
+					"lines": "Error: timeout",
+				})
+				continue
+
 			exit_code = handle.poll()
 			if exit_code is None:
-				continue
+				print(f"[Zipgrep] Timeout for {package.name}", file=sys.stderr)
+				handle.kill()
+				results.append({
+					"package": package.as_key_dict(),
+					"lines": "Error: timeout",
+				})
 			elif exit_code == 0:
+				print(f"[Zipgrep] Success for {package.name}", file=sys.stderr)
 				results.append({
 					"package": package.as_key_dict(),
 					"lines": handle.stdout.read(),
 				})
-
-			del running[i]
-
-		# Create new
-		while len(running) < 1 and len(packages) > 0:
-			package = packages.pop()
-			release: Optional[PackageRelease] = package.get_download_release()
-			if release:
-				handle = Popen(["zipgrep", query, release.file_path, file_filter], stdout=PIPE, encoding="UTF-8")
-				running.append([package, handle])
-
-		if len(running) > 0:
-			running[0][1].wait()
+			elif exit_code != 1:
+				print(f"[Zipgrep] Error {exit_code} for {package.name}", file=sys.stderr)
+				results.append({
+					"package": package.as_key_dict(),
+					"lines": f"Error: exit {exit_code}",
+				})
 
 	return {
 		"query": query,

app/template_filters.py

@@ -23,7 +23,7 @@ from flask_login import current_user
 from markupsafe import Markup
 
 from . import app, utils
-from .markdown import get_headings
+from app.markdown import get_headings
 from .models import Permission, Package, PackageState, PackageRelease
 from .utils import abs_url_for, url_set_query, url_set_anchor, url_current
 from .utils.minetest_hypertext import normalize_whitespace as do_normalize_whitespace

app/templates/base.html

@@ -16,7 +16,7 @@
 	{%- endif %}
 
 	<link rel="stylesheet" type="text/css" href="/static/libs/bootstrap.min.css?v=4">
-	<link rel="stylesheet" type="text/css" href="/static/custom.css?v=54">
+	<link rel="stylesheet" type="text/css" href="/static/custom.css?v=55">
 	<link rel="search" type="application/opensearchdescription+xml" href="/static/opensearch.xml" title="ContentDB" />
 
 	{% if noindex -%}

app/templates/index.html

@@ -39,24 +39,25 @@
 		</div>
 		<div class="carousel-inner">
 			{% for package in spotlight_pkgs %}
+				{% set meta = package.get_translated(load_desc=False) %}
 				{% set cover_image = package.get_cover_image_url() %}
 				{% set tags =  package.tags | sort(attribute="views", reverse=True) %}
 				<div class="carousel-item {% if loop.index == 1 %}active{% endif %}">
 					<a href="{{ package.get_url('packages.view') }}">
 						<div class="ratio ratio-16x9">
 							<img src="{{ cover_image }}"
-								alt="{{ _('%(title)s by %(author)s', title=package.title, author=package.author.display_name) }}">
+								alt="{{ _('%(title)s by %(author)s', title=meta.title, author=package.author.display_name) }}">
 						</div>
 						<div class="carousel-caption text-shadow">
 							<h3 class="mt-0 mb-3">
 								{% if package.author %}
-									{{ _('<strong>%(title)s</strong> by %(author)s', title=package.title, author=package.author.display_name) }}
+									{{ _('<strong>%(title)s</strong> by %(author)s', title=meta.title, author=package.author.display_name) }}
 								{% else %}
-									<strong>{{ package.title }}</strong>
+									<strong>{{ meta.title }}</strong>
 								{% endif %}
 							</h3>
 							<p>
-								{{ package.short_desc }}
+								{{ meta.short_desc }}
 							</p>
 							{% if package.author %}
 								<div class="d-none d-md-block">

app/templates/packages/create_edit.html

@@ -18,7 +18,7 @@
 	{{ form_scripts() }}
 	{{ easymde_scripts() }}
 	{% if enable_wizard %}
-		<script src="/static/js/polltask.js"></script>
+		<script src="/static/js/polltask.js?v=3"></script>
 		<script src="/static/js/package_create.js"></script>
 	{% endif %}
 	<script src="/static/js/package_edit.js?v=3"></script>

app/templates/tasks/view.html

@@ -10,18 +10,17 @@
 
 {% block content %}
 	<h1>{{ self.title() }}</h1>
+	<article data-task-id="{{ info.id }}">
+		<p id="status"></p>
+		<div id="progress" class="progress d-none">
+			<div class="progress-bar bg-info" role="progressbar" style="width: 50%;" aria-valuenow="50" aria-valuemin="0" aria-valuemax="100"></div>
+		</div>
+	</article>
 
 	{% if "error" in info or info.status == "FAILURE" or info.status == "REVOKED" %}
 		<pre style="white-space: pre-wrap; word-wrap: break-word;">{{ info.error }}</pre>
 	{% else %}
-		<script src="/static/js/polltask.js"></script>
-		<script>
-			// @author rubenwardy
-			// @license magnet:?xt=urn:btih:1f739d935676111cfff4b4693e3816e664797050&dn=gpl-3.0.txt GPL-v3-or-Later
-			pollTask("{{ url_for('tasks.check', id=info.id) }}", true)
-					.then(function() { location.reload() })
-					.catch(function() { location.reload() })
-		</script>
+		<script src="/static/js/polltask.js?v=3"></script>
 		<noscript>
 			{{ _("Reload the page to check for updates.") }}
 		</noscript>

app/templates/zipgrep/search.html

@@ -17,6 +17,7 @@
 		{{ form.hidden_tag() }}
 		{{ render_field(form.query, hint=self.query_hint()) }}
 		{{ render_field(form.file_filter, hint="Supports wildcards and regex") }}
+		{{ render_field(form.type, hint=_("Use shift to select multiple. Leave selection empty to match any type.")) }}
 		{{ render_submit_field(form.submit, tabindex=180) }}
 	</form>
 

requirements.lock.txt

@@ -40,7 +40,9 @@ kombu==5.3.7
 libsass==0.23.0
 lxml==5.2.2
 Mako==1.3.5
-Markdown==3.6
+markdown-it-py==3.0.0
+mdit-py-plugins==0.4.2
+linkify-it-py==2.0.3
 MarkupSafe==2.1.5
 packaging==24.0
 passlib==1.7.4

requirements.txt

@@ -10,7 +10,9 @@ GitHub-Flask
 SQLAlchemy-Searchable
 
 bcrypt
-markdown
+markdown-it-py
+linkify-it-py
+mdit-py-plugins
 bleach
 passlib