From 8f622ba5c908425e10eaa986562fae5bc9fc9fe9 Mon Sep 17 00:00:00 2001 From: rubenwardy Date: Sat, 2 Apr 2022 20:09:59 +0100 Subject: [PATCH] Add ability to search for text in all packages --- app/blueprints/tasks/__init__.py | 3 +- app/blueprints/zipgrep/__init__.py | 66 +++++++++++++++++++++++++ app/tasks/zipgrep.py | 62 +++++++++++++++++++++++ app/templates/zipgrep/search.html | 26 ++++++++++ app/templates/zipgrep/view_results.html | 47 ++++++++++++++++++ 5 files changed, 203 insertions(+), 1 deletion(-) create mode 100644 app/blueprints/zipgrep/__init__.py create mode 100644 app/tasks/zipgrep.py create mode 100644 app/templates/zipgrep/search.html create mode 100644 app/templates/zipgrep/view_results.html diff --git a/app/blueprints/tasks/__init__.py b/app/blueprints/tasks/__init__.py index bb2cea5a..0753a42e 100644 --- a/app/blueprints/tasks/__init__.py +++ b/app/blueprints/tasks/__init__.py @@ -25,6 +25,7 @@ from app.utils import * bp = Blueprint("tasks", __name__) + @csrf.exempt @bp.route("/tasks/getmeta/new/", methods=["POST"]) @login_required @@ -36,6 +37,7 @@ def start_getmeta(): "poll_url": url_for("tasks.check", id=aresult.id), }) + @bp.route("/tasks//") def check(id): result = celery.AsyncResult(id) @@ -43,7 +45,6 @@ def check(id): traceback = result.traceback result = result.result - None if isinstance(result, Exception): info = { 'id': id, diff --git a/app/blueprints/zipgrep/__init__.py b/app/blueprints/zipgrep/__init__.py new file mode 100644 index 00000000..e80d2575 --- /dev/null +++ b/app/blueprints/zipgrep/__init__.py @@ -0,0 +1,66 @@ +# ContentDB +# Copyright (C) 2022 rubenwardy +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see . + + +from celery import uuid +from flask import Blueprint, render_template, redirect, request +from flask_wtf import FlaskForm +from wtforms import StringField, BooleanField, SubmitField +from wtforms.validators import InputRequired, Length + +from app.tasks import celery +from app.utils import rank_required + +bp = Blueprint("zipgrep", __name__) + +from app.models import * +from app.tasks.zipgrep import search_in_releases + + +class SearchForm(FlaskForm): + query = StringField(lazy_gettext("Text to find (regex)"), [InputRequired(), Length(6, 100)]) + file_filter = StringField(lazy_gettext("File filter"), [InputRequired(), Length(1, 100)], default="*.lua") + remember_me = BooleanField(lazy_gettext("Remember me"), default=True) + submit = SubmitField(lazy_gettext("Search")) + + +@bp.route("/zipgrep/", methods=["GET", "POST"]) +@rank_required(UserRank.ADMIN) +def zipgrep_search(): + form = SearchForm(request.form) + if form.validate_on_submit(): + task_id = uuid() + search_in_releases.apply_async((form.query.data, form.file_filter.data), task_id=task_id) + result_url = url_for("zipgrep.view_results", id=task_id) + return redirect(url_for("tasks.check", id=task_id, r=result_url)) + + return render_template("zipgrep/search.html", form=form) + + +@bp.route("/zipgrep//") +def view_results(id): + result = celery.AsyncResult(id) + if result.status != "SUCCESS" or isinstance(result.result, Exception): + result_url = url_for("zipgrep.view_results", id=id) + return redirect(url_for("tasks.check", id=id, r=result_url)) + + matches = result.result["matches"] + for match in matches: + match["package"] = Package.query.filter( + Package.name == match["package"]["name"], + Package.author.has(username=match["package"]["author"])).one() + + return render_template("zipgrep/view_results.html", query=result.result["query"], matches=matches) diff --git a/app/tasks/zipgrep.py b/app/tasks/zipgrep.py new file mode 100644 index 00000000..81dd81c3 --- /dev/null +++ b/app/tasks/zipgrep.py @@ -0,0 +1,62 @@ +# ContentDB +# Copyright (C) 2022 rubenwardy +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see . + + +import subprocess +from subprocess import Popen, PIPE +from typing import Optional + +from app.models import Package, PackageState, PackageRelease +from app.tasks import celery + + +@celery.task() +def search_in_releases(query: str, file_filter: str): + packages = list(Package.query.filter(Package.state == PackageState.APPROVED).all()) + running = [] + results = [] + + while len(packages) > 0 or len(running) > 0: + # Check running + for i in range(len(running) - 1, -1, -1): + package: Package = running[i][0] + handle: subprocess.Popen[str] = running[i][1] + exit_code = handle.poll() + if exit_code is None: + continue + elif exit_code == 0: + results.append({ + "package": package.getAsDictionaryKey(), + "lines": handle.stdout.read(), + }) + + del running[i] + + # Create new + while len(running) < 1 and len(packages) > 0: + package = packages.pop() + release: Optional[PackageRelease] = package.getDownloadRelease() + if release: + handle = Popen(["zipgrep", query, release.file_path, file_filter], stdout=PIPE, encoding="UTF-8") + running.append([package, handle]) + + if len(running) > 0: + running[0][1].wait() + + return { + "query": query, + "matches": results, + } diff --git a/app/templates/zipgrep/search.html b/app/templates/zipgrep/search.html new file mode 100644 index 00000000..0a40859a --- /dev/null +++ b/app/templates/zipgrep/search.html @@ -0,0 +1,26 @@ +{% extends "base.html" %} + +{% block title %} +{{ _("Search in Package Releases") }} +{% endblock %} + +{% block query_hint %} + + POSIX Extended Regular Expressions + +{% endblock %} + +{% block content %} +

{{ self.title() }}

+ {% from "macros/forms.html" import render_field, render_submit_field %} +
+ {{ form.hidden_tag() }} + {{ render_field(form.query, hint=self.query_hint()) }} + {{ render_field(form.file_filter, hint="Supports wildcards and regex") }} + {{ render_submit_field(form.submit, tabindex=180) }} +
+ +

+ For more information, see ZipGrep's man page. +

+{% endblock %} diff --git a/app/templates/zipgrep/view_results.html b/app/templates/zipgrep/view_results.html new file mode 100644 index 00000000..08df29dd --- /dev/null +++ b/app/templates/zipgrep/view_results.html @@ -0,0 +1,47 @@ +{% extends "base.html" %} + +{% block title %} +{{ _("'%(query)s' - Search Package Releases", query=query) }} +{% endblock %} + +{% block content %} + New Query +

{{ _("Search in Package Releases") }}

+

{{ query }}

+ +

+ Found in {{ matches | count }} packages. +

+ +
+ {% for match in matches %} +
+
+
+ + +
+ + {{ match.package.title }} + + by {{ match.package.author.display_name }} +
+ +

+ {{ match.lines.split("\n") | select | list | count }} matches +

+ + + Download + +
+
+ 
{{ match.lines }}
+
+
+
+ {% endfor %} +
+{% endblock %}