Add skip button to importer

This reverts commit dd6257a0a0.

.gitignore

@@ -7,6 +7,7 @@ log.txt
 *.rdb
 uploads
 thumbnails
+celerybeat-schedule
 
 # Created by https://www.gitignore.io/api/linux,macos,python,windows
 

app/flatpages/help/wtfpl.md

@@ -0,0 +1,42 @@
+title: WTFPL is a terrible license
+no_h1: true
+
+<div id="warning" class="box box_grey alert alert-warning">
+	<span class="icon_message"></span>
+
+	Please reconsider the choice of WTFPL as a license.
+
+	<script src="/static/jquery.min.js"></script>
+	<script>
+		// @author rubenwardy
+		// @license magnet:?xt=urn:btih:1f739d935676111cfff4b4693e3816e664797050&dn=gpl-3.0.txt GPL-v3-or-Later
+
+		var params = new URLSearchParams(location.search);
+		var r      = params.get("r");
+		if (r)
+			document.write("<a class='alert_right button' href='" + r + "'>Okay</a>");
+		else
+			$("#warning").hide();
+	</script>
+</div>
+
+# WTFPL is a terrible license
+
+The use of WTFPL as a license is discouraged for multiple reasons.
+
+* **No Warranty disclaimer:** This could open you up to being sued.<sup>[1]</sup>
+* **Swearing:** This prevents settings like schools from using your content.
+* **Not OSI Approved:** Same as public domain?
+
+The Open Source Initiative chose not to approve the license as an open-source
+license, saying:<sup>[3]</sup>
+
+> It's no different from dedication to the public domain.
+> Author has submitted license approval request – author is free to make public domain dedication.
+> Although he agrees with the recommendation, Mr. Michlmayr notes that public domain doesn't exist in Europe. Recommend: Reject.
+
+## Sources
+
+1. [WTFPL is harmful to software developers](https://cubicspot.blogspot.com/2017/04/wtfpl-is-harmful-to-software-developers.html)
+2. [FSF](https://www.gnu.org/licenses/license-list.en.html)
+3. [OSI](https://opensource.org/minutes20090304)

app/flatpages/policy_and_guidance.md

@@ -65,8 +65,7 @@ package and give the name to the correct one.
 If you submit a package where you don't have the right to the name you will be asked
 to change the name of the package, or your package won't be accepted.
 
-We reserve the right to issue exceptions for this where we feel necessary, however
+We reserve the right to issue exceptions for this where we feel necessary.
-this will be done rarely and usually only for packages created before CDB was created.
 
 ### 3.2 Mod Forks and Reimplementations
 
@@ -108,8 +107,9 @@ It is recommended that you use a proper license for code with a warranty
 disclaimer, such as the (L)GPL or MIT. You should also use a proper media license
 for media, such as a Creative Commons license.
 
-The use of WTFPL is discouraged as it doesn't contain a valid warranty disclaimer,
+The use of WTFPL is discouraged as it doesn't contain a [valid warranty disclaimer](https://cubicspot.blogspot.com/2017/04/wtfpl-is-harmful-to-software-developers.html),
-and also includes swearing which dissuades teachers from using your content.
+and also includes swearing which prevents settings like schools from using your content.
+[Read more](/help/wtfpl/).
 
 Public domain is not a valid license in many countries, please use CC0 or MIT instead.
 

app/models.py

@@ -393,7 +393,7 @@ class Package(db.Model):
 			"name": self.name,
 			"title": self.title,
 			"author": self.author.display_name,
-			"shortDesc": self.shortDesc,
+			"short_description": self.shortDesc,
 			"type": self.type.toName(),
 			"release": self.getDownloadRelease().id if self.getDownloadRelease() is not None else None,
 			"thumbnail": (base_url + tnurl) if tnurl is not None else None,
@@ -406,17 +406,17 @@ class Package(db.Model):
 			"author": self.author.display_name,
 			"name": self.name,
 			"title": self.title,
-			"shortDesc": self.shortDesc,
+			"short_description": self.shortDesc,
 			"desc": self.desc,
 			"type": self.type.toName(),
-			"createdAt": self.created_at,
+			"created_at": self.created_at,
 
 			"license": self.license.name,
-			"mediaLicense": self.media_license.name,
+			"media_license": self.media_license.name,
 
 			"repo": self.repo,
 			"website": self.website,
-			"issueTracker": self.issueTracker,
+			"issue_tracker": self.issueTracker,
 			"forums": self.forums,
 
 			"provides": [x.name for x in self.provides],
@@ -514,17 +514,21 @@ class Package(db.Model):
 	def recalcScore(self):
 		import datetime
 
-		self.score = 0
+		self.score = 10
 
-		if self.forums is None:
+		if self.forums is not None:
-			return
+			topic = ForumTopic.query.get(self.forums)
+			if topic:
+				days   = (datetime.datetime.now() - topic.created_at).days
+				months = days / 30
+				years  = days / 365
+				self.score = topic.views / max(years, 0.0416) + 80*min(max(months, 0.5), 6)
 
-		topic = ForumTopic.query.get(self.forums)
+		if self.getMainScreenshotURL() is None:
-		if topic:
+			self.score *= 0.8
-			days   = (datetime.datetime.now() - topic.created_at).days
+
-			months = days / 30
+		if not self.license.is_foss or not self.media_license.is_foss:
-			years  = days / 365
+			self.score *= 0.1
-			self.score = topic.views / years + 80*min(6, months)
 
 class MetaPackage(db.Model):
 	id           = db.Column(db.Integer, primary_key=True)

app/public/static/package_create.js

@@ -11,6 +11,8 @@ $(function() {
 
 	$(".pkg_meta").hide()
 	$(".pkg_wiz_1").show()
+
+	$("#pkg_wiz_1_skip").click(finish)
 	$("#pkg_wiz_1_next").click(function() {
 		const repoURL = $("#repo").val();
 		if (repoURL.trim() != "") {

app/scss/components.scss

@@ -459,3 +459,15 @@ table.fancyTable tfoot td {
 .wiptopic a {
 	color: #7ac;
 }
+
+.editor-toolbar {
+	background-color: #333 !important;
+}
+
+.CodeMirror {
+	background-color: #222 !important;
+}
+
+.editor-preview-side {
+	background-color: #222 !important;
+}

app/tasks/__init__.py

@@ -18,6 +18,7 @@
 import flask
 from flask.ext.sqlalchemy import SQLAlchemy
 from celery import Celery
+from celery.schedules import crontab
 from app import app
 from app.models import *
 
@@ -64,4 +65,12 @@ def make_celery(app):
 
 celery = make_celery(app)
 
+CELERYBEAT_SCHEDULE = {
+	'topic_list_import': {
+		'task': 'app.tasks.forumtasks.importTopicList',
+		'schedule': crontab(minute=1, hour=1),
+	}
+}
+celery.conf.beat_schedule = CELERYBEAT_SCHEDULE
+
 from . import importtasks, forumtasks, emails

app/tasks/forumtasks.py

@@ -74,7 +74,7 @@ def parseTitle(title):
 def getLinksFromModSearch():
 	links = {}
 
-	contents = urllib.request.urlopen("http://krock-works.16mb.com/MTstuff/modList.php").read().decode("utf-8")
+	contents = urllib.request.urlopen("https://krock-works.uk.to/minetest/modList.php").read().decode("utf-8")
 	for x in json.loads(contents):
 		link = x.get("link")
 		if link is not None:
@@ -127,15 +127,18 @@ def importTopicList():
 		link = links_by_id.get(id)
 
 		# Fill row
-		topic.topic_id   = id
+		topic.topic_id   = int(id)
 		topic.author     = user
 		topic.type       = info["type"]
 		topic.title      = title
 		topic.name       = name
 		topic.link       = link
 		topic.wip        = info["wip"]
-		topic.posts      = info["posts"]
+		topic.posts      = int(info["posts"])
-		topic.views      = info["views"]
+		topic.views      = int(info["views"])
 		topic.created_at = info["date"]
 
+	for p in Package.query.all():
+		p.recalcScore()
+
 	db.session.commit()

app/tasks/importtasks.py

@@ -66,7 +66,7 @@ def getKrockList():
 	global krock_list_cache_by_name
 
 	if krock_list_cache is None:
-		contents = urllib.request.urlopen("http://krock-works.16mb.com/MTstuff/modList.php").read().decode("utf-8")
+		contents = urllib.request.urlopen("https://krock-works.uk.to/minetest/modList.php").read().decode("utf-8")
 		list = json.loads(contents)
 
 		def h(x):
@@ -149,7 +149,8 @@ class PackageTreeNode:
 			type = PackageType.GAME
 		elif os.path.isfile(baseDir + "/init.lua"):
 			type = PackageType.MOD
-		elif os.path.isfile(baseDir + "/modpack.txt"):
+		elif os.path.isfile(baseDir + "/modpack.txt") or \
+				os.path.isfile(baseDir + "/modpack.conf"):
 			type = PackageType.MOD
 			is_modpack = True
 		elif os.path.isdir(baseDir + "/mods"):

app/templates/flatpage.html

@@ -5,7 +5,7 @@
 {% endblock %}
 
 {% block content %}
-	<h1>{{ page['title'] }}</h1>
+	{% if not page["no_h1"] %}<h1>{{ page['title'] }}</h1>{% endif %}
 
 	{{ page.html | safe }}
 {% endblock %}

app/templates/index.html

@@ -12,8 +12,8 @@ Welcome
 		<p>
 			Minetest's official content repository.
 			Browse {{ count }} packages,
-			majority of which available under a free and open source
+			the majority of which are available under a free
-			license.
+			and open source license.
 		</p>
 
 		<form method="get" action="/packages/">

app/templates/macros/topics.html

@@ -1,4 +1,4 @@
-{% macro render_topictable(topics, show_author=True) -%}
+{% macro render_topics_table(topics, show_author=True) -%}
 <table>
 	<tr>
 		<th>Id</th>
@@ -31,3 +31,22 @@
 	{% endfor %}
 </table>
 {% endmacro %}
+
+
+{% macro render_topics(topics, current_user, show_author=True) -%}
+<ul>
+	{% for topic in topics %}
+		<li{% if topic.wip %} class="wiptopic"{% endif %}>
+			<a href="https://forum.minetest.net/viewtopic.php?t={{ topic.topic_id}}">{{ topic.title }}</a>
+			{% if topic.wip %}[WIP]{% endif %}
+			{% if topic.name %}[{{ topic.name }}]{% endif %}
+			{% if show_author %}
+				by <a href="{{ url_for('user_profile_page', username=topic.author.username) }}">{{ topic.author.display_name }}</a>
+			{% endif %}
+			{% if topic.author == current_user or topic.author.checkPerm(current_user, "CHANGE_AUTHOR") %}
+				| <a href="{{ url_for('create_edit_package_page', author=topic.author.username, repo=topic.getRepoURL(), forums=topic.topic_id, title=topic.title, bname=topic.name) }}">Create</a>
+			{% endif %}
+		</li>
+	{% endfor %}
+</ul>
+{% endmacro %}

app/templates/packages/create_edit.html

@@ -51,13 +51,14 @@
 			<p>Enter the repo URL for the package.
 			If the repo uses git then the metadata will be automatically imported.</p>
 
-			<p>Leave blank if you don't have a repo.</p>
+			<p>Leave blank if you don't have a repo. Click skip if the import fails.</p>
 		</div>
 
 		{{ render_field(form.repo, class_="pkg_repo") }}
 
 		<div class="pkg_wiz_1">
-			<a id="pkg_wiz_1_next" class="button button-primary">Next</a>
+			<a id="pkg_wiz_1_next" class="button button-primary">Next (Autoimport)</a>
+			<a id="pkg_wiz_1_skip" class="button button-default">Skip Autoimport</a>
 		</div>
 
 		<div class="pkg_wiz_2">
@@ -70,6 +71,12 @@
 		<div class="pkg_meta">{{ render_submit_field(form.submit) }}</div>
 	</form>
 
+	<script src="/static/simplemde.min.js"></script>
+	<link rel="stylesheet" type="text/css" href="/static/simplemde.min.css">
+	<script>
+		var simplemde = new SimpleMDE({ element: $("#desc")[0] });
+	</script>
+
 	{% if enable_wizard %}
 		<script src="/static/url.min.js"></script>
 		<script src="/static/polltask.js"></script>

app/templates/packages/list.html

@@ -9,6 +9,7 @@
 		{% if type %}<input type="hidden" name="type" value="{{ type }}" />{% endif %}
 		<input type="text" name="q" value="{{ query or ''}}" />
 		<input type="submit" value="Search" />
+		<input type="submit" name="lucky" value="I'm feeling lucky" />
 
 		<p>
 			Found {{ packages_count }} packages.
@@ -37,4 +38,12 @@
 		<li>{{ page }} / {{ page_max }}</li>
 		{% if next_url %}<li><a href="{{ next_url }}">Next</a></li> {% endif %}
 	</ul>
+
+	{% if topics %}
+		<h2 style="margin-top:2em;">More content from the forums</h2>
+
+		{% from "macros/topics.html" import render_topics %}
+		{{ render_topics(topics, current_user) }}
+	{% endif %}
+
 {% endblock %}

app/templates/todo/topics.html

@@ -12,6 +12,6 @@ Topics to be Added
 		{{ topics | count }} remaining.
 	</p>
 
-	{% from "macros/topictable.html" import render_topictable %}
+	{% from "macros/topics.html" import render_topics_table %}
-	{{ render_topictable(topics) }}
+	{{ render_topics_table(topics) }}
 {% endblock %}

app/templates/users/user_profile_page.html

@@ -107,8 +107,8 @@
 				List of your forum topics which do not have a matching package.
 			</p>
 
-			{% from "macros/topictable.html" import render_topictable %}
+			{% from "macros/topics.html" import render_topics_table %}
-			{{ render_topictable(topics_to_add, show_author=False) }}
+			{{ render_topics_table(topics_to_add, show_author=False) }}
 		</div>
 	</div>
 {% endif %}

app/utils.py

@@ -170,3 +170,13 @@ def clearNotifications(url):
 	if current_user.is_authenticated:
 		Notification.query.filter_by(user=current_user, url=url).delete()
 		db.session.commit()
+
+
+YESES = ["yes", "true", "1", "on"]
+
+def isYes(val):
+	return val and val.lower() in YESES
+
+
+def isNo(val):
+	return val and not isYes(val)

app/views/api.py

@@ -20,11 +20,13 @@ from flask_user import *
 from app import app
 from app.models import *
 from app.utils import is_package_page
-from .packages import build_packages_query
+from .packages import QueryBuilder
 
 @app.route("/api/packages/")
 def api_packages_page():
-	query, _ = build_packages_query()
+	qb    = QueryBuilder()
+	query = qb.buildPackageQuery()
+
 	pkgs = [package.getAsDictionaryShort(app.config["BASE_URL"]) \
 			for package in query.all() if package.getDownloadRelease() is not None]
 	return jsonify(pkgs)

app/views/githublogin.py

@@ -51,9 +51,9 @@ def github_authorized(oauth_token):
 	if current_user and current_user.is_authenticated:
 		if userByGithub is None:
 			current_user.github_username = username
-			db.session.add(auth)
 			db.session.commit()
-			return redirect(url_for("gitAccount", id=auth.id))
+			flash("Linked github to account", "success")
+			return redirect(url_for("home_page"))
 		else:
 			flash("Github account is already associated with another user", "danger")
 			return redirect(url_for("home_page"))

app/views/packages/__init__.py

@@ -31,28 +31,66 @@ from wtforms.validators import *
 from wtforms.ext.sqlalchemy.fields import QuerySelectField, QuerySelectMultipleField
 from sqlalchemy import or_, any_
 
-def build_packages_query():
-	title = "Packages"
 
-	query = Package.query.filter_by(soft_deleted=False, approved=True)
+class QueryBuilder:
+	title  = None
+	types  = None
+	search = None
 
-	# Filter by requested type(s)
+	def __init__(self):
-	types = request.args.getlist("type")
+		title = "Packages"
-	types = [PackageType.get(tname) for tname in types]
-	types = [type for type in types if type is not None]
-	if len(types) > 0:
-		title = ", ".join([type.value + "s" for type in types])
 
-		query = query.filter(Package.type.in_(types))
+		# Get request types
+		types = request.args.getlist("type")
+		types = [PackageType.get(tname) for tname in types]
+		types = [type for type in types if type is not None]
+		if len(types) > 0:
+			title = ", ".join([type.value + "s" for type in types])
 
+		self.title  = title
+		self.types  = types
+		self.search = request.args.get("q")
+		self.lucky  = "lucky" in request.args
+		self.hide_nonfree = isNo(request.args.get("nonfree"))
+		self.limit  = 1 if self.lucky else None
 
-	search = request.args.get("q")
+	def buildPackageQuery(self):
-	if search is not None and search.strip() != "":
+		query = Package.query.filter_by(soft_deleted=False, approved=True)
-		query = query.filter(Package.title.ilike('%' + search + '%'))
 
-	query = query.order_by(db.desc(Package.score))
+		if len(self.types) > 0:
+			query = query.filter(Package.type.in_(self.types))
 
-	return query, title
+		if self.search is not None and self.search.strip() != "":
+			query = query.filter(Package.title.ilike('%' + self.search + '%'))
+
+		query = query.order_by(db.desc(Package.score))
+
+		if self.hide_nonfree:
+			query = query.filter(Package.license.has(License.is_foss == True))
+			query = query.filter(Package.media_license.has(License.is_foss == True))
+
+		if self.limit:
+			query = query.limit(self.limit)
+
+		return query
+
+	def buildTopicQuery(self):
+		topics = ForumTopic.query \
+				.filter(~ db.exists().where(Package.forums==ForumTopic.topic_id)) \
+				.order_by(db.asc(ForumTopic.wip), db.asc(ForumTopic.name), db.asc(ForumTopic.title)) \
+				.filter(ForumTopic.title.ilike('%' + self.search + '%'))
+
+		if len(self.types) > 0:
+			topics = topics.filter(ForumTopic.type.in_(self.types))
+
+		if self.hide_nonfree:
+			query = query.filter(Package.license.has(License.is_foss == True))
+			query = query.filter(Package.media_license.has(License.is_foss == True))
+
+		if self.limit:
+			topics = topics.limit(self.limit)
+
+		return topics
 
 @menu.register_menu(app, ".mods", "Mods", order=11, endpoint_arguments_constructor=lambda: { 'type': 'mod' })
 @menu.register_menu(app, ".games", "Games", order=12, endpoint_arguments_constructor=lambda: { 'type': 'game' })
@@ -62,7 +100,19 @@ def packages_page():
 	if shouldReturnJson():
 		return redirect(url_for("api_packages_page"))
 
-	query, title = build_packages_query()
+	qb    = QueryBuilder()
+	query = qb.buildPackageQuery()
+	title = qb.title
+
+	if qb.lucky:
+		package = query.first()
+		if package:
+			return redirect(package.getDetailsURL())
+
+		topic = qb.buildTopicQuery().first()
+		if topic:
+			return redirect("https://forum.minetest.net/viewtopic.php?t=" + str(topic.topic_id))
+
 	page  = int(request.args.get("page") or 1)
 	num   = min(42, int(request.args.get("n") or 100))
 	query = query.paginate(page, num, True)
@@ -75,8 +125,13 @@ def packages_page():
 	prev_url = url_for("packages_page", type=type_name, q=search, page=query.prev_num) \
 			if query.has_prev else None
 
+	topics = None
+	if qb.search and not query.has_next:
+		topics = qb.buildTopicQuery().all()
+
 	tags = Tag.query.all()
-	return render_template("packages/list.html", title=title, packages=query.items, \
+	return render_template("packages/list.html", \
+			title=title, packages=query.items, topics=topics, \
 			query=search, tags=tags, type=type_name, \
 			next_url=next_url, prev_url=prev_url, page=page, page_max=query.pages, packages_count=query.total)
 
@@ -279,11 +334,15 @@ def create_edit_package_page(author=None, name=None):
 
 		db.session.commit() # save
 
+		next_url = package.getDetailsURL()
 		if wasNew and package.repo is not None:
 			task = importRepoScreenshot.delay(package.id)
-			return redirect(url_for("check_task", id=task.id, r=package.getDetailsURL()))
+			next_url = url_for("check_task", id=task.id, r=next_url)
 
-		return redirect(package.getDetailsURL())
+		if wasNew and ("WTFPL" in package.license.name or "WTFPL" in package.media_license.name):
+			next_url = url_for("flatpage", path="help/wtfpl", r=next_url)
+
+		return redirect(next_url)
 
 	package_query = Package.query.filter_by(approved=True, soft_deleted=False)
 	if package is not None:

app/views/tagseditor.py

@@ -27,7 +27,7 @@ from app.utils import rank_required
 @app.route("/tags/")
 @rank_required(UserRank.MODERATOR)
 def tag_list_page():
-	return render_template("admin/tagslist.html", tags=Tag.query.order_by(db.asc(Tag.title)).all())
+	return render_template("admin/tags/list.html", tags=Tag.query.order_by(db.asc(Tag.title)).all())
 
 class TagForm(FlaskForm):
 	title	 = StringField("Title", [InputRequired(), Length(3,100)])

requirements.txt

@@ -6,7 +6,7 @@ Flask-Menu>=0.7.0
 Flask-Markdown>=0.3
 GitHub-Flask>=3.2.0
 pyScss==1.3.4
-celery==4.0.2
+celery==4.1.1
 redis==2.10.6
 beautifulsoup4==4.6.0
 lxml==4.2.1