v0.12.0: newer Python and pips

Author: 1fish2

borealis/docker_task.py

@@ -282,7 +282,8 @@ def _terminate(self, container, logger, reason, terminated):
         and the Docker client.
 
         NOTE: "The KeyboardInterrupt exception will be received by an arbitrary
-        thread." -- https://docs.python.org/3.8/library/_thread.html
+        thread. (When the signal module is available, interrupts always go to
+        the main thread.)" -- https://docs.python.org/3.8/library/_thread.html
         """
         name = self['name']
         logger.debug('Terminating task {} for {}...'.format(name, reason))

borealis/setup/how-to-install-gce-server.txt

@@ -107,7 +107,7 @@ echo y | gcloud auth configure-docker
 ## level Image that's common to the team (containing runtime components like
 ## Linux apts, Python, and pips), then preinstalling the runtime Image will
 ## save Docker Image build time as well as task startup time.
-### Example: docker pull python:3.8.9
+### Example: docker pull python:3.11.3
 ### Example: docker pull gcr.io/$(gcloud config get-value core/project)/my-app-runtime
 
 
@@ -138,10 +138,10 @@ sudo su -l fireworker
 
 mkdir ~/borealis && cd ~/borealis
 
-pyenv install 3.8.9
+pyenv install 3.11.3
 
-pyenv global 3.8.9
-pyenv local 3.8.9
+pyenv global 3.11.3
+pyenv local 3.11.3
 pip install --upgrade pip setuptools virtualenv virtualenvwrapper virtualenv-clone wheel
 
 pyenv virtualenv fireworker
@@ -262,7 +262,7 @@ sudo systemctl stop borealis-fireworker
 sudo journalctl --vacuum-time=2d
 
 ## Pull any Docker images that you want to preload for speed.
-docker pull python:3.8.9  # or similar image(s) including your project's runtime Image
+docker pull python:3.11.3  # or similar image(s) including your project's runtime Image
 
 ## Prune Docker images:
 docker image prune

borealis/setup/requirements.txt

@@ -1,57 +1,57 @@
-# Direct requirements
-# dnspython is needed for pymongo to access server clusters.
-dnspython==2.1.0
-docker==5.0.0
-FireWorks==1.9.7
-google-cloud-logging==2.3.1
-google-cloud-storage==1.37.1
-requests==2.25.1
-ruamel.yaml==0.17.4
-ruamel.yaml.clib==0.2.2
+# Direct requirements with frozen versions for reproducible testing.
+# FireWorks requires pymongo; dnspython enables pymongo to access server clusters.
+dnspython==2.3.0
+docker==6.0.1
+FireWorks==2.0.3
+google-cloud-logging==3.5.0
+google-cloud-storage==2.8.0
+requests==2.28.2
+ruamel.yaml==0.17.21
+ruamel.yaml.clib==0.2.7
 
-# Indirect requirements
-appdirs==1.4.4
-cachetools==4.2.1
-certifi==2020.12.5
-cffi==1.14.5
-chardet==4.0.0
-click==7.1.2
-distlib==0.3.1
-filelock==3.0.12
-Flask==1.1.2
-flask-paginate==0.8.1
-google-api-core==1.26.3
-google-auth==1.29.0
-google-cloud-core==1.6.0
-google-crc32c==1.1.2
-google-resumable-media==1.2.0
-googleapis-common-protos==1.53.0
-grpcio==1.37.0
+# Indirect requirements added by pip freeze.
+cachetools==5.3.0
+certifi==2022.12.7
+charset-normalizer==3.1.0
+click==8.1.3
+distlib==0.3.6
+filelock==3.11.0
+Flask==2.2.3
+flask-paginate==2022.1.8
+google-api-core==2.11.0
+google-auth==2.17.2
+google-cloud-appengine-logging==1.3.0
+google-cloud-audit-log==0.2.5
+google-cloud-core==2.3.2
+google-crc32c==1.5.0
+google-resumable-media==2.4.1
+googleapis-common-protos==1.59.0
+grpc-google-iam-v1==0.12.6
+grpcio==1.53.0
+grpcio-status==1.53.0
 gunicorn==20.1.0
-idna==2.10
-itsdangerous==1.1.0
-Jinja2==2.11.3
-MarkupSafe==1.1.1
-monty==2021.3.3
-packaging==20.9
-pbr==5.5.1
-proto-plus==1.18.1
-protobuf==3.15.8
+idna==3.4
+itsdangerous==2.1.2
+Jinja2==3.1.2
+MarkupSafe==2.1.2
+monty==2022.9.9
+packaging==23.0
+pbr==5.11.1
+platformdirs==3.2.0
+proto-plus==1.22.2
+protobuf==4.22.1
 pyasn1==0.4.8
 pyasn1-modules==0.2.8
-pycparser==2.20
-pymongo==3.11.3
-pyparsing==2.4.7
-python-dateutil==2.8.1
-pytz==2021.1
-rsa==4.7.2
-six==1.15.0
-stevedore==3.3.0
-tabulate==0.8.9
-tqdm==4.60.0
-urllib3==1.26.5
-virtualenv==20.4.3
-virtualenv-clone==0.5.4
+pymongo==4.3.3
+python-dateutil==2.8.2
+rsa==4.9
+six==1.16.0
+stevedore==5.0.0
+tabulate==0.9.0
+tqdm==4.65.0
+urllib3==1.26.15
+virtualenv==20.21.0
+virtualenv-clone==0.5.7
 virtualenvwrapper==4.8.4
-websocket-client==0.58.0
-Werkzeug==1.0.1
+websocket-client==1.5.1
+Werkzeug==2.2.3

docs/changes.md

@@ -1,5 +1,8 @@
 # Change Log
 
+## v0.12.0
+* Update the installation instructions to use Python 3.11.3 and newer pip libraries.
+
 ## v0.11.0
 * In the `gce create` CLI command, implement LaunchPad config `uri_mode` by setting the `host`, `uri_mode`, and `db` metadata fields.
 

docs/developer-setup.md

@@ -126,7 +126,7 @@ statement to your shell `.profile` or `.bash_profile` file:
 
    Then run that `export` command or create a new shell.
 
-1. In a fresh development directory, install Python 3.8, create a Python
+1. In a fresh development directory, install Python 3.11, create a Python
 virtual environment, and install the borealis-fireworks pip.
 
 1. Create or augment your `my_launchpad.yaml` file:

docs/docker-build.md

@@ -13,7 +13,7 @@ First write a `Dockerfile` in your project directory. Example:
 # The ARG lets you pick another base Image but DON'T USE AN ALPINE base since
 # its floating point library produces different results; also
 # see https://pythonspeed.com/articles/alpine-docker-python/
-ARG from=python:3.8.6
+ARG from=python:3.8.16
 FROM ${from}
 
 # Install Linux packages.
@@ -59,7 +59,7 @@ docker build --tag application101 .
 You can base it on a different "FROM" Image like this:
 
 ```shell script
-docker build --build-arg from=python:3.8.1 --tag application101 .
+docker build --build-arg from=python:3.11.3 --tag application101 .
 ```
 
 but **don't use an Alpine base image** because its floating point library produces

docs/install-tools.md

@@ -7,12 +7,12 @@ Also see [Handy Links](handy-links.md).
 
 1. Install Python and Fireworks:
 
-   * Install Python: `pyenv install 3.8.6`
+   * Install Python: `pyenv install 3.11.3`
    * Install a Python virtual environment manager like `virtualenv`,
      `pyenv-virtualenv`, and `pyenv-virtualenvwrapper`; or `venv`
    * Create a Python virtual environment in your project directory, e.g.:
      ```shell script
-     pyenv local 3.8.6
+     pyenv local 3.11.3
      pyenv virtualenv myproject
      pyenv local myproject
      ```
@@ -27,21 +27,21 @@ Also see [Handy Links](handy-links.md).
 
 1. Setup Python for the `gcloud` and `gsutil` tools.
 
-   **Note:** `gsutil -m` is broken in Python 3.8 so use Python 3.6 or 3.7.
+   **Note:** `gsutil -m` is still broken in Python 3.8+ so use Python 3.6 or 3.7.
    ([gsutil issue #961](https://github.com/GoogleCloudPlatform/gsutil/issues/961).)
 
    Install Python 3.6 or 3.7 if you don't already have one of them, e.g.
    using `pyenv`:
 
    ```shell script
-   pyenv install 3.7.9
+   pyenv install 3.7.16
    ```
 
    Set `$CLOUDSDK_PYTHON` in your shell profile:
 
    ```shell script
    # Set the Python version for Cloud SDK.
-   export CLOUDSDK_PYTHON=$(pyenv shell 3.7.9; pyenv which python)
+   export CLOUDSDK_PYTHON=$(pyenv shell 3.7.16; pyenv which python)
    ```
 
    Then open a new shell (or run this `export` command).

requirements.txt

@@ -1,19 +1,21 @@
 ## Create the "fireworker" python virtual environment for development, and select it in this directory:
-#   pyenv install 3.8.9
+#   pyenv install 3.11.3
 #
-#   pyenv virtualenv 3.8.9 fireworker && pyenv local fireworker
+#   pyenv virtualenv 3.11.3 fireworker && pyenv local fireworker
 #   pip install --upgrade pip setuptools virtualenv virtualenvwrapper virtualenv-clone wheel
 #   pip install -r requirements.txt && pyenv rehash
 
-# Direct dependents
-# dnspython is needed for pymongo to access server clusters.
-dnspython==2.1.0
-google-cloud-logging>=2.3.1
-google-cloud-storage>=1.37.1
-docker>=5.0.0
-FireWorks>=1.9.6
-requests>=2.25.1
-ruamel.yaml>=0.17.4
+# Direct requirements.
+# FireWorks requires pymongo; dnspython enables pymongo to access server clusters.
+# The binary extension ruamel.yaml.clib helps ruamel.yaml run faster, right?
+dnspython
+google-cloud-logging
+google-cloud-storage
+docker
+FireWorks
+requests
+ruamel.yaml
+ruamel.yaml.clib
 
 # Dev tools
 twine

setup.py

@@ -15,7 +15,7 @@
 
 setup(
     name='borealis-fireworks',
-    version='0.11.0',
+    version='0.12.0',
     packages=['borealis', 'borealis.util'],
     url='https://github.com/CovertLab/borealis',
     project_urls={
@@ -34,7 +34,7 @@
         'google-cloud-logging>=2.0.0',
         'google-cloud-storage>=1.28.0',
         'docker>=4.1.0',
-        'FireWorks>=1.9.5',
+        'FireWorks>=1.9.7',
         'requests>=2.22.0',
         'ruamel.yaml>=0.16.9',
     ],
@@ -58,6 +58,9 @@
         'Programming Language :: Python',
         'Programming Language :: Python :: 3 :: Only',
         'Programming Language :: Python :: 3.8',
+        'Programming Language :: Python :: 3.9',
+        'Programming Language :: Python :: 3.10',
+        'Programming Language :: Python :: 3.11',
     ],
     keywords='fireworks workflow',
 )