Triplehx3/contentdb
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Fix various issues with forum topic importing

Browse Source 
Fixes #201
This commit is contained in:
rubenwardy
2024-06-22 11:11:57 +01:00
parent 12545c69ac
commit ca961cb35f
5 changed files with 68 additions and 42 deletions
Download Patch File Download Diff File Expand all files Collapse all files

7
app/utils/phpbbparser.py
View File

@@ -3,6 +3,7 @@
# Source: https://github.com/rubenwardy/python_phpbb_parser
import re
import sys
import urllib
import urllib.parse as urlparse
import urllib.request
@@ -121,7 +122,7 @@ regex_id = re.compile(r"^.*t=([0-9]+).*$")
def parse_forum_list_page(id, page, out, extra=None):
num_per_page = 30
start = page*num_per_page+1
print(" - Fetching page {} (topics {}-{})".format(page, start, start+num_per_page))
print(" - Fetching page {} (topics {}-{})".format(page, start, start+num_per_page), file=sys.stderr)
url = "https://forum.minetest.net/viewforum.php?f=" + str(id) + "&start=" + str(start)
r = urllib.request.urlopen(url).read().decode("utf-8")
@@ -154,7 +155,7 @@ def parse_forum_list_page(id, page, out, extra=None):
views = topic.find(class_="views").find(text=True)
if id in out:
print(" - got {} again, title: {}".format(id, title))
print(" - got {} again, title: {}".format(id, title), file=sys.stderr)
assert title == out[id]['title']
return False
@@ -177,7 +178,7 @@ def parse_forum_list_page(id, page, out, extra=None):
def get_topics_from_forum(id, out, extra=None):
print("Fetching all topics from forum {}".format(id))
print("Fetching all topics from forum {}".format(id), file=sys.stderr)
page = 0
while parse_forum_list_page(id, page, out, extra):
page = page + 1