Triplehx3/contentdb
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Clean up code

Browse Source
This commit is contained in:
rubenwardy
2020-12-04 02:23:04 +00:00
parent 0c0d3e1715
commit 42f96618e2
62 changed files with 254 additions and 558 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
app/tasks/__init__.py
View File

@@ -16,10 +16,8 @@
import flask
from flask_sqlalchemy import SQLAlchemy
from celery import Celery
from celery.schedules import crontab
from app import app
from app.models import *
class TaskError(Exception):

2
app/tasks/emails.py
View File

@@ -15,7 +15,7 @@
# along with this program. If not, see <https://www.gnu.org/licenses/>.
from flask import render_template, url_for
from flask import render_template
from flask_mail import Message
from app import mail
from app.tasks import celery

5
app/tasks/forumtasks.py
View File

@@ -15,14 +15,11 @@
# along with this program. If not, see <https://www.gnu.org/licenses/>.
import flask, json, re
from flask_sqlalchemy import SQLAlchemy
from app import app
import json, re
from app.models import *
from app.tasks import celery
from .phpbbparser import getProfile, getTopicsFromForum
import urllib.request
from urllib.parse import urlparse, quote_plus
@celery.task()
def checkForumAccount(username, forceNoSave=False):

20
app/tasks/importtasks.py
View File

@@ -15,22 +15,17 @@
# along with this program. If not, see <https://www.gnu.org/licenses/>.
import flask, json, os, git, tempfile, shutil, gitdb, contextlib
import os, git, tempfile, shutil, gitdb, contextlib
from git import GitCommandError
from git_archive_all import GitArchiver
from flask_sqlalchemy import SQLAlchemy
from urllib.error import HTTPError
import urllib.request
from urllib.parse import urlparse, quote_plus, urlsplit
from urllib.parse import urlsplit
from zipfile import ZipFile
from app import app
from app.models import *
from app.tasks import celery, TaskError
from app.utils import randomString, getExtension
from .minetestcheck import build_tree, MinetestCheckError, ContentType
from .minetestcheck.config import parse_conf
from .krocklist import getKrockList, findModInfo
def generateGitURL(urlstr):
@@ -60,7 +55,7 @@ def cloneRepo(urlstr, ref=None, recursive=False):
print("Cloning from " + gitUrl)
if ref is None:
repo = git.Repo.clone_from(gitUrl, gitDir, \
repo = git.Repo.clone_from(gitUrl, gitDir,
progress=None, env=None, depth=1, recursive=recursive, kill_after_timeout=15)
else:
repo = git.Repo.init(gitDir)
@@ -96,10 +91,7 @@ def getMeta(urlstr, author):
except MinetestCheckError as err:
raise TaskError(str(err))
result = {}
result["name"] = tree.name
result["provides"] = tree.getModNames()
result["type"] = tree.type.name
result = {"name": tree.name, "provides": tree.getModNames(), "type": tree.type.name}
for key in ["depends", "optional_depends"]:
result[key] = tree.fold("meta", key)
@@ -120,8 +112,8 @@ def getMeta(urlstr, author):
def postReleaseCheckUpdate(self, release, path):
try:
tree = build_tree(path, expected_type=ContentType[release.package.type.name], \
author=release.package.author.username, name=release.package.name)
tree = build_tree(path, expected_type=ContentType[release.package.type.name],
author=release.package.author.username, name=release.package.name)
cache = {}
def getMetaPackages(names):

80
app/tasks/krocklist.py
View File

@@ -1,80 +0,0 @@
# ContentDB
# Copyright (C) 2018-20 rubenwardy
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program. If not, see <https://www.gnu.org/licenses/>.
import json
import urllib.request
krock_list_cache = None
krock_list_cache_by_name = None
def getKrockList():
global krock_list_cache
global krock_list_cache_by_name
if krock_list_cache is None:
contents = urllib.request.urlopen("https://krock-works.uk.to/minetest/modList.php").read().decode("utf-8")
list = json.loads(contents)
def h(x):
if not ("title" in x and "author" in x and \
"topicId" in x and "link" in x and x["link"] != ""):
return False
import re
m = re.search(r"\[([A-Za-z0-9_]+)\]", x["title"])
if m is None:
return False
x["name"] = m.group(1)
return True
def g(x):
return {
"title": x["title"],
"author": x["author"],
"name": x["name"],
"topicId": x["topicId"],
"link": x["link"],
}
krock_list_cache = [g(x) for x in list if h(x)]
krock_list_cache_by_name = {}
for x in krock_list_cache:
if not x["name"] in krock_list_cache_by_name:
krock_list_cache_by_name[x["name"]] = []
krock_list_cache_by_name[x["name"]].append(x)
return krock_list_cache, krock_list_cache_by_name
def findModInfo(author, name, link):
list, lookup = getKrockList()
if name is not None and name in lookup:
if len(lookup[name]) == 1:
return lookup[name][0]
for x in lookup[name]:
if x["author"] == author:
return x
if link is not None and len(link) > 15:
for x in list:
if link in x["link"]:
return x
return None

4
app/tasks/minetestcheck/__init__.py
View File

@@ -20,7 +20,7 @@ class ContentType(Enum):
"""
Whether or not `other` is an acceptable type for this
"""
assert(other)
assert other
if self == ContentType.MOD:
if not other.isModLike():
@@ -40,7 +40,7 @@ def build_tree(path, expected_type=None, author=None, repo=None, name=None):
path = get_base_dir(path)
root = PackageTreeNode(path, "/", author=author, repo=repo, name=name)
assert(root)
assert root
if expected_type:
expected_type.validate_same(root.type)

19
app/tasks/minetestcheck/tree.py
View File

@@ -54,12 +54,12 @@ class PackageTreeNode:
if self.type == ContentType.GAME:
if not os.path.isdir(baseDir + "/mods"):
raise MinetestCheckError(("Game at {} does not have a mods/ folder").format(self.relative))
raise MinetestCheckError("Game at {} does not have a mods/ folder".format(self.relative))
self.add_children_from_mod_dir("mods")
elif self.type == ContentType.MOD:
if self.name and not basenamePattern.match(self.name):
raise MinetestCheckError(("Invalid base name for mod {} at {}, names must only contain a-z0-9_.") \
.format(self.name, self.relative))
raise MinetestCheckError("Invalid base name for mod {} at {}, names must only contain a-z0-9_." \
.format(self.name, self.relative))
elif self.type == ContentType.MODPACK:
self.add_children_from_mod_dir(None)
@@ -132,11 +132,12 @@ class PackageTreeNode:
for dep in result["depends"]:
if not basenamePattern.match(dep):
if " " in dep:
raise MinetestCheckError(("Invalid dependency name '{}' for mod at {}, did you forget a comma?") \
raise MinetestCheckError("Invalid dependency name '{}' for mod at {}, did you forget a comma?" \
.format(dep, self.relative))
else:
raise MinetestCheckError(("Invalid dependency name '{}' for mod at {}, names must only contain a-z0-9_.") \
.format(dep, self.relative))
raise MinetestCheckError(
"Invalid dependency name '{}' for mod at {}, names must only contain a-z0-9_." \
.format(dep, self.relative))
# Check dependencies
@@ -177,11 +178,11 @@ class PackageTreeNode:
if not entry.startswith('.') and os.path.isdir(path):
child = PackageTreeNode(path, relative + entry + "/", name=entry)
if not child.type.isModLike():
raise MinetestCheckError(("Expecting mod or modpack, found {} at {} inside {}") \
.format(child.type.value, child.relative, self.type.value))
raise MinetestCheckError("Expecting mod or modpack, found {} at {} inside {}" \
.format(child.type.value, child.relative, self.type.value))
if child.name is None:
raise MinetestCheckError(("Missing base name for mod at {}").format(self.relative))
raise MinetestCheckError("Missing base name for mod at {}".format(self.relative))
self.children.append(child)

17
app/tasks/phpbbparser.py
View File

@@ -2,15 +2,14 @@
# License: MIT
# Source: https://github.com/rubenwardy/python_phpbb_parser
import urllib, socket
from bs4 import *
from urllib.parse import urljoin
from datetime import datetime
import urllib.request
import os.path
import time, re
import re
import urllib
import urllib.parse as urlparse
import urllib.request
from datetime import datetime
from urllib.parse import urlencode
from bs4 import *
def urlEncodeNonAscii(b):
return re.sub('[\x80-\xFF]', lambda c: '%%%02x' % ord(c.group(0)), b)
@@ -68,7 +67,7 @@ def __extract_properties(profile, soup):
def __extract_signature(soup):
res = soup.find_all("div", class_="signature")
if (len(res) != 1):
if len(res) != 1:
return None
else:
return res[0]
@@ -166,7 +165,7 @@ def parseForumListPage(id, page, out, extra=None):
return True
def getTopicsFromForum(id, out={}, extra=None):
def getTopicsFromForum(id, out, extra=None):
print("Fetching all topics from forum {}".format(id))
page = 0
while parseForumListPage(id, page, out, extra):