Merge pull request #71 from pomo-mondreganto/fixes

Fix some stuff, upgrade deps

Author: pomo-mondreganto

.github/workflows/release.yml

@@ -27,7 +27,7 @@ jobs:
         with:
           registry: ghcr.io
           username: ${{ github.repository_owner }}
-          password: ${{ secrets.GITHUB_TOKEN }}
+          password: ${{ github.token }}
 
       - name: Build and push
         id: docker_build

.github/workflows/tests.yml

@@ -17,14 +17,11 @@ jobs:
   tests:
     strategy:
       matrix:
-        os:
-          - ubuntu-20.04
-          - ubuntu-18.04
         test_type:
           - BLITZ
           - CLASSIC
 
-    runs-on: ${{ matrix.os }}
+    runs-on: "ubuntu-20.04"
     timeout-minutes: 15
     env:
       TEST: 1
@@ -41,7 +38,7 @@ jobs:
       - uses: actions/cache@v2
         with:
           path: ~/.cache/pip
-          key: ${{ runner.os }}-pip-${{ hashFiles('**/requirements.txt') }}
+          key: ${{ runner.os }}-pip-${{ hashFiles('backend/requirements.txt') }}-${{ hashFiles('cli/requirements.txt') }}-${{ hashFiles('tests/requirements.txt') }}
           restore-keys: |
             ${{ runner.os }}-pip-
       - name: Setup tests

Makefile

@@ -1,7 +1,14 @@
-.PHONY: lint
-lint:
+.PHONY: lint-backend
+lint-backend:
 	flake8 --config .flake8
 
+.PHONY: lint-frontend
+lint-frontend:
+	cd front && npx eslint .
+
+.PHONY: lint
+lint: lint-backend lint-frontend
+
 .PHONY: clean
 clean:
 	./control.py reset || :
@@ -23,4 +30,4 @@ release-base:
 start: clean
 	./control.py setup
 	./control.py start --fast
-	./control.py rd logs -f inititializer
+	./control.py rd logs -f initializer

README.md

@@ -54,7 +54,7 @@ statistics, services description, writing a checker, modifying the rating system
    images or backend requirements, omit this option to run the full build.
 
 That's all! Now you should be able to access the scoreboard at `http://127.0.0.1:8080/`. Admin panel is accessible at
-`http://127.0.0.1:8080/admin`. Celery visualization (flower) is at `http://127.0.0.1:8080/flower`.
+`http://127.0.0.1:8080/admin/`. Celery visualization (flower) is at `http://127.0.0.1:8080/flower/`.
 
 > Before each new game run `./control.py reset` to delete old database and temporary files (and docker networks)
 
@@ -92,7 +92,7 @@ Config file (`config.yml`) is split into five main parts:
 
     * `default_score` (optional, default `2500`): default score for tasks.
 
-    * `env_path` (optional, default `/checkers/bin/`): string to append to checkers' `$PATH` environment variable
+    * `env_path` (optional): string to append to checkers' `$PATH` environment variable
       (see [checkers](#checkers) section). Example: `/checkers/bin/`.
 
     * `game_hardness` (optional, default `10`): game hardness parameter
@@ -109,6 +109,8 @@ Config file (`config.yml`) is split into five main parts:
     * `username: forcad`
     * `password: **change_me**`
 
+    It will be auto-generated if missing. Usernames & passwords to all storages will be the same as to the admin panel.
+
 * **teams** contains playing teams. Example contents:
 
 ```yaml
@@ -128,7 +130,7 @@ Highlighted teams will be marked on the scoreboard with a rainbow border.
 ```yaml
 tasks:
   - checker: collacode/checker.py
-    checker_type: gevent_pfr
+    checker_type: pfr
     checker_timeout: 30
     default_score: 1500
     gets: 3
@@ -145,7 +147,7 @@ tasks:
     puts: 2
 ```
 
-* **storages** is an **auto-generated section**, which will be overridden by `control.py setup/kube setup` and describes
+* **storages** is an **auto-generated section**, which will be overridden by `control.py <setup>/<kube setup>` and describes
   settings used to connect to PostgreSQL, Redis and RabbitMQ:
 
     * `db`: PostgreSQL settings:
@@ -206,7 +208,7 @@ The following options are supported:
 
 * `checker_type` (optional, default `hackerdom`): an option containing underscore-separated tags,
   (missing tags are ignored). Examples: `hackerdom` (hackerdom tag ignored, so no modifier tags are applied),
-  `gevent_pfr` (gevent checker with public flag data returned). Currently, supported tags are:
+  `pfr` (checker with public flag data returned). Currently, supported tags are:
 
     * `pfr`: checker returns public flag data (e.g. username of flag user) from `PUT` action as a **public message**,
       private flag data (`flag_id`) as a **private message**, and **public message** is shown
@@ -217,16 +219,11 @@ The following options are supported:
       random generator in checkers so it would return the same values for `GET` and `PUT`. Checkers supporting this
       options are quite rare (and old), so **don't use it** unless you're sure.
 
-    * `gevent`: an experimental checker type to make checkers faster. **Don't use it** if you're not absolutely sure you
-      know how it works. **Don't use it** on long and (or) large competitions! Example checker
-      is [here](tests/service/checker/gevent_checker.py).
-
 More detailed explanation of checker tags can be
 found [in this issue](https://github.com/pomo-mondreganto/ForcAD/issues/18#issuecomment-618072993).
 
-* `env_path` (optional, default `/checkers/bin`): path or a combination of paths to be prepended to `PATH` env
-  variable (e.g. path to chromedriver). By default, `/checkers/bin` is used, so all auxiliary executables can be but
-  in `checkers/bin`.
+* `env_path` (optional): path or a combination of paths to be prepended to `PATH` env
+  variable (e.g. path to chromedriver).
 
 See more in [checker writing](#writing-a-checker) section.
 
@@ -236,11 +233,11 @@ See more in [checker writing](#writing-a-checker) section.
 
 ```yaml
 checkers:
-  - requirements.txt  <--   automatically installed (with pip) combined requirements of all checkers
+  - requirements.txt  <--   automatically installed (with pip) combined requirements of all checkers (must be present)
   - task1:
-      - checker.py  <--   executable
+      - checker.py  <--   executable (o+rx)
   - task2:
-      - checker.py  <--   executable
+      - checker.py  <--   executable (o+rx)
 ```
 
 ### Writing a checker

backend/lib/config/models.py

@@ -39,10 +39,11 @@ class Celery(BaseModel):
 
     worker_prefetch_multiplier: int = 1
 
+    result_expires = 15 * 60
     redis_socket_timeout: int = 10
     redis_socket_keepalive: bool = True
     redis_retry_on_timeout: bool = True
 
-    accept_content: List[str] = ['pickle']
+    accept_content: List[str] = ['pickle', 'json']
     result_serializer: str = 'pickle'
     task_serializer: str = 'pickle'

backend/lib/helpers/__init__.py

@@ -1,7 +1,7 @@
 from . import (
     cache, checkers, commands,
     events, exceptions, jobs,
-    thread_actions, singleton,
+    singleton,
 )
 
 __all__ = (
@@ -11,6 +11,5 @@
     'events',
     'exceptions',
     'jobs',
-    'thread_actions',
     'singleton',
 )

backend/lib/helpers/checkers.py

@@ -1,20 +1,9 @@
 from logging import Logger
-from typing import Optional, List
+from typing import Optional
 
 from lib import models
 from lib.helpers.commands import run_generic_command
-from lib.helpers.thread_actions import run_generic_action_in_thread
-from lib.models import TaskStatus, Action
-
-
-def first_error_or_first_verdict(
-        verdicts: List[models.CheckerVerdict],
-) -> models.CheckerVerdict:
-    for verdict in verdicts:
-        if verdict.status != TaskStatus.UP:
-            return verdict
-
-    return verdicts[0]
+from lib.models import Action
 
 
 class CheckerRunner:
@@ -37,18 +26,12 @@ def __init__(
         self.flag = flag
 
     def check(self) -> models.CheckerVerdict:
-        if self.task.is_checker_gevent_optimized:
-            return self._check_in_thread()
         return self._check_as_process()
 
     def put(self) -> models.CheckerVerdict:
-        if self.task.is_checker_gevent_optimized:
-            return self._put_in_thread()
         return self._put_as_process()
 
     def get(self) -> models.CheckerVerdict:
-        if self.task.is_checker_gevent_optimized:
-            return self._get_in_thread()
         return self._get_as_process()
 
     def _check_as_process(self) -> models.CheckerVerdict:
@@ -108,53 +91,3 @@ def _get_as_process(self) -> models.CheckerVerdict:
             team=self.team,
             logger=self.logger,
         )
-
-    def _check_in_thread(self) -> models.CheckerVerdict:
-        """Check implementation, gevent-compatible"""
-
-        return run_generic_action_in_thread(
-            action=Action.CHECK,
-            task=self.task,
-            team=self.team,
-            logger=self.logger,
-            action_args=(),
-            action_kwargs={},
-        )
-
-    def _put_in_thread(self) -> models.CheckerVerdict:
-        """Check implementation, gevent-compatible"""
-        assert self.flag is not None, 'Can only be called when flag is passed'
-
-        kwargs = {
-            'flag_id': self.flag.private_flag_data,
-            'flag': self.flag.flag,
-            'vuln': str(self.flag.vuln_number),
-        }
-
-        return run_generic_action_in_thread(
-            action=Action.PUT,
-            task=self.task,
-            team=self.team,
-            logger=self.logger,
-            action_args=(),
-            action_kwargs=kwargs,
-        )
-
-    def _get_in_thread(self) -> models.CheckerVerdict:
-        """Check implementation, gevent-compatible"""
-        assert self.flag is not None, 'Can only be called when flag is passed'
-
-        kwargs = {
-            'flag_id': self.flag.private_flag_data,
-            'flag': self.flag.flag,
-            'vuln': str(self.flag.vuln_number),
-        }
-
-        return run_generic_action_in_thread(
-            action=Action.GET,
-            task=self.task,
-            team=self.team,
-            logger=self.logger,
-            action_args=(),
-            action_kwargs=kwargs,
-        )

backend/lib/helpers/exceptions.py

@@ -2,10 +2,6 @@ class FlagSubmitException(Exception):
     """Exception for flag validation."""
 
 
-class CheckerTimeoutException(BaseException):
-    """Exception raised by gevent-optimized checkers."""
-
-
 class FlagExceptionEnum:
     GAME_NOT_AVAILABLE = FlagSubmitException('Game is not available.')
     FLAG_INVALID = FlagSubmitException('Flag is invalid or too old.')