sync CI/build scripts with other CWL projects

Author: mr-c

.github/dependabot.yml

@@ -15,3 +15,8 @@ updates:
   - dependency-name: mypy
     versions:
     - "0.800"
+  # Maintain dependencies for GitHub Actions
+  - package-ecosystem: "github-actions"
+    directory: "/"
+    schedule:
+      interval: "daily"

.github/workflows/ci-tests.yml

@@ -5,7 +5,10 @@ on:
     branches: [ main ]
   pull_request:
     branches: [ main ]
-  workflow_dispatch:
+
+concurrency:
+  group: build-${{ github.event.pull_request.number || github.ref }}
+  cancel-in-progress: true
 
 env:
   singularity_version: 3.6.4
@@ -20,8 +23,8 @@ jobs:
     strategy:
       matrix:
         py-ver-major: [3]
-        py-ver-minor: [6, 7, 8, 9]
-        step: [lint, unit, mypy]
+        py-ver-minor: [6, 7, 8, 9, 10]
+        step: [lint, unit, bandit, mypy]
 
     env:
       py-semver: ${{ format('{0}.{1}', matrix.py-ver-major, matrix.py-ver-minor) }}
@@ -38,10 +41,6 @@ jobs:
         with:
           singularity-version: ${{ env.singularity_version }}
 
-      - name: Give the test runner user a name to make provenance happy.
-        if: ${{ matrix.step == 'unit' || matrix.step == 'mypy' }}
-        run: sudo usermod -c 'CI Runner' $(whoami)
-
       - name: Set up Python
         uses: actions/setup-python@v2
         with:
@@ -70,7 +69,7 @@ jobs:
 
       - name: Upload coverage to Codecov
         if: ${{ matrix.step == 'unit' }}
-        uses: codecov/codecov-action@v1
+        uses: codecov/codecov-action@v2.1.0
         with:
           fail_ci_if_error: true
 
@@ -84,8 +83,8 @@ jobs:
         step: [lint-readme, pydocstyle]
 
     env:
-      py-semver: 3.9
-      TOXENV: ${{ format('py39-{0}', matrix.step) }}
+      py-semver: "3.10"
+      TOXENV: ${{ format('py310-{0}', matrix.step) }}
 
     steps:
       - uses: actions/checkout@v2
@@ -115,7 +114,6 @@ jobs:
       - name: Test with tox
         run: tox
 
-
   release_test:
     name: cwl-utils release test
 
@@ -132,10 +130,7 @@ jobs:
       - name: Set up Python
         uses: actions/setup-python@v2
         with:
-          python-version: 3.9
-
-      - name: Give the test runner user a name to make provenance happy.
-        run: sudo usermod -c 'CI Runner' $(whoami)
+          python-version: "3.10"  # quoted, otherwise that turns into the number 3.1
 
       - name: Cache for pip
         uses: actions/cache@v2

.github/workflows/codeql-analysis.yml

@@ -6,6 +6,10 @@ on:
   schedule:
     - cron: '0 0 * * 0'
 
+concurrency:
+  group: build-${{ github.event.pull_request.number || github.ref }}
+  cancel-in-progress: true
+
 jobs:
   CodeQL-Build:
 

Makefile

@@ -21,21 +21,23 @@
 # make coverage-report to check coverage of the python scripts by the tests
 
 MODULE=cwl_utils
+PACKAGE=cwl-utils
+EXTRAS=
 
 # `SHELL=bash` doesn't work for some, so don't use BASH-isms like
 # `[[` conditional expressions.
 PYSOURCES=$(filter-out parser/cwl_v%,$(shell find $(MODULE) -name "*.py")) $(wildcard tests/*.py) *.py
-DEVPKGS=diff_cover black pylint coverage pep257 pydocstyle flake8 mypy\
-	isort wheel autoflake
+DEVPKGS=diff_cover black pylint pep257 pydocstyle flake8 tox tox-pyenv \
+	isort wheel autoflake flake8-bugbear pyupgrade bandit \
+	-rtest-requirements.txt -rmypy-requirements.txt
 DEBDEVPKGS=pep8 python-autopep8 pylint python-coverage pydocstyle sloccount \
 	   python-flake8 python-mock shellcheck
 VERSION=$(shell awk '{print $3}' < cwl_utils/__meta__.py )
 mkfile_dir := $(dir $(abspath $(lastword $(MAKEFILE_LIST))))
 UNAME_S=$(shell uname -s)
 
 ## all         : default task
-all:
-	pip install -e .
+all: dev
 
 ## help        : print this help message and exit
 help: Makefile
@@ -46,20 +48,19 @@ install-dep: install-dependencies
 
 install-dependencies:
 	pip install --upgrade $(DEVPKGS)
-	pip install -r requirements.txt -r mypy_requirements.txt -r docs/requirements.txt
+	pip install -r requirements.txt -r mypy-requirements.txt -r docs/requirements.txt
 
 ## install-deb-dep: install most of the dev dependencies via apt-get
 install-deb-dep:
 	sudo apt-get install $(DEBDEVPKGS)
 
-## install     : install the ${MODULE} module and schema-salad-tool
+## install     : install the ${MODULE} module and scripts
 install: FORCE
-	pip install .[deps]
+	pip install .$(EXTRAS)
 
 ## dev     : install the ${MODULE} module in dev mode
 dev: install-dep
-	pip install -e .[deps]
-
+	pip install -e .$(EXTRAS)
 
 ## dist        : create a module package for distribution
 dist: dist/${MODULE}-$(VERSION).tar.gz
@@ -101,12 +102,15 @@ diff_pydocstyle_report: pydocstyle_report.txt
 format: $(PYSOURCES)
 	black $^
 
+format-check: $(PYSOURCES)
+	black --diff --check $^
+
 ## pylint      : run static code analysis on Python code
 pylint: $(PYSOURCES)
 	pylint --msg-template="{path}:{line}: [{msg_id}({symbol}), {obj}] {msg}" \
                 $^ -j0|| true
 
-pylint_report.txt: ${PYSOURCES}
+pylint_report.txt: $(PYSOURCES)
 	pylint --msg-template="{path}:{line}: [{msg_id}({symbol}), {obj}] {msg}" \
 		$^ -j0> $@ || true
 
@@ -138,17 +142,17 @@ diff-cover.html: coverage.xml
 
 ## test        : run the ${MODULE} test suite
 test: $(PYSOURCES)
-	pytest # --addopts "-n auto --dist=loadfile"
+	python setup.py test ${PYTEST_EXTRA}
 
 ## testcov     : run the ${MODULE} test suite and collect coverage
 testcov: $(PYSOURCES)
-	pytest --cov ${MODULE} # -n auto --dist=loadfile"
+	python setup.py test --addopts "--cov" ${PYTEST_EXTRA}
 
-sloccount.sc: ${PYSOURCES} Makefile
+sloccount.sc: $(PYSOURCES) Makefile
 	sloccount --duplicates --wide --details $^ > $@
 
 ## sloccount   : count lines of code
-sloccount: ${PYSOURCES} Makefile
+sloccount: $(PYSOURCES) Makefile
 	sloccount $^
 
 list-author-emails:
@@ -165,17 +169,24 @@ mypy: $(filter-out setup.py,${PYSOURCES})
 	fi  # if minimally required ruamel.yaml version is 0.15.99 or greater, than the above can be removed
 	MYPYPATH=$$MYPYPATH:typeshed mypy $^
 
+pyupgrade: $(PYSOURCES)
+	pyupgrade --exit-zero-even-if-changed --py36-plus $^
+
 release-test: FORCE
 	git diff-index --quiet HEAD -- || ( echo You have uncommited changes, please commit them and try again; false )
 	./release-test.sh
 
 release: release-test
 	. testenv2/bin/activate && \
-		python testenv2/src/${MODULE}/setup.py sdist bdist_wheel && \
+		python testenv2/src/${PACKAGE}/setup.py sdist bdist_wheel
+	. testenv2/bin/activate && \
 		pip install twine && \
-		twine upload testenv2/src/${MODULE}/dist/* && \
+		twine upload testenv2/src/${PACKAGE}/dist/* && \
 		git tag ${VERSION} && git push --tags
 
+flake8: $(PYSOURCES)
+	flake8 $^
+
 FORCE:
 
 # Use this to print the value of a Makefile variable

README.md README.rstREADME.md renamed to README.rst

@@ -7,11 +7,25 @@
 import shutil
 import sys
 from pathlib import Path
-from typing import Any, Callable, Dict, List, MutableSequence, Optional, Tuple, Union
+from typing import (
+    Any,
+    Callable,
+    Dict,
+    List,
+    MutableSequence,
+    Optional,
+    Tuple,
+    Union,
+    TYPE_CHECKING,
+)
 
 from cwltool.loghandler import _logger as _cwltoollogger
 from ruamel import yaml
-from typing_extensions import Protocol
+
+if TYPE_CHECKING:
+    from typing_extensions import Protocol
+else:
+    Protocol = object
 
 _logger = logging.getLogger("cwl-expression-refactor")  # pylint: disable=invalid-name
 defaultStreamHandler = logging.StreamHandler()  # pylint: disable=invalid-name

cwl_utils/cwl_expression_refactor.py

@@ -42,7 +42,7 @@ def expand_stream_shortcuts(process: cwl.CommandLineTool) -> cwl.CommandLineTool
             stdout_path = process.stdout
             if not stdout_path:
                 stdout_path = str(
-                    hashlib.sha1(
+                    hashlib.sha1(  # nosec
                         json_dumps(cwl.save(process)).encode("utf-8")
                     ).hexdigest()
                 )
@@ -101,7 +101,7 @@ def get_expression(
             "cores": 0,
             "ram": 0,
             "outdir": "/root",
-            "tmpdir": "/tmp",
+            "tmpdir": "/tmp",  # nosec
             "outdirSize": 0,
             "tmpdirSize": 0,
         }
@@ -1446,8 +1446,8 @@ def traverse_CommandLineTool(
                         "basename": "base.name",
                         "nameroot": "base",
                         "nameext": "name",
-                        "path": "/tmp/base.name",
-                        "dirname": "/tmp",
+                        "path": "/tmp/base.name",  # nosec
+                        "dirname": "/tmp",  # nosec
                     }
                 ]
                 if outp.outputBinding.loadContents:

cwl_utils/cwl_v1_0_expression_refactor.py

@@ -42,7 +42,7 @@ def expand_stream_shortcuts(process: cwl.CommandLineTool) -> cwl.CommandLineTool
             stdout_path = process.stdout
             if not stdout_path:
                 stdout_path = str(
-                    hashlib.sha1(
+                    hashlib.sha1(  # nosec
                         json_dumps(cwl.save(process)).encode("utf-8")
                     ).hexdigest()
                 )
@@ -101,7 +101,7 @@ def get_expression(
             "cores": 0,
             "ram": 0,
             "outdir": "/root",
-            "tmpdir": "/tmp",
+            "tmpdir": "/tmp",  # nosec
             "outdirSize": 0,
             "tmpdirSize": 0,
         }
@@ -1446,8 +1446,8 @@ def traverse_CommandLineTool(
                         "basename": "base.name",
                         "nameroot": "base",
                         "nameext": "name",
-                        "path": "/tmp/base.name",
-                        "dirname": "/tmp",
+                        "path": "/tmp/base.name",  # nosec
+                        "dirname": "/tmp",  # nosec
                     }
                 ]
                 if outp.outputBinding.loadContents:

cwl_utils/cwl_v1_1_expression_refactor.py

@@ -42,7 +42,7 @@ def expand_stream_shortcuts(process: cwl.CommandLineTool) -> cwl.CommandLineTool
             stdout_path = process.stdout
             if not stdout_path:
                 stdout_path = str(
-                    hashlib.sha1(
+                    hashlib.sha1(  # nosec
                         json_dumps(cwl.save(process)).encode("utf-8")
                     ).hexdigest()
                 )
@@ -101,7 +101,7 @@ def get_expression(
             "cores": 0,
             "ram": 0,
             "outdir": "/root",
-            "tmpdir": "/tmp",
+            "tmpdir": "/tmp",  # nosec
             "outdirSize": 0,
             "tmpdirSize": 0,
         }
@@ -1542,8 +1542,8 @@ def traverse_CommandLineTool(
                         "basename": "base.name",
                         "nameroot": "base",
                         "nameext": "name",
-                        "path": "/tmp/base.name",
-                        "dirname": "/tmp",
+                        "path": "/tmp/base.name",  # nosec
+                        "dirname": "/tmp",  # nosec
                     }
                 ]
                 if outp.outputBinding.loadContents:

cwl_utils/cwl_v1_2_expression_refactor.py

@@ -2,7 +2,7 @@
 """Classes for docker-extract."""
 import logging
 import os
-import subprocess
+import subprocess  # nosec
 from abc import ABC, abstractmethod
 from typing import List
 
@@ -27,7 +27,7 @@ def save_docker_image(self) -> None:
     @staticmethod
     def _run_command_pull(cmd_pull: List[str]) -> None:
         try:
-            subprocess.run(
+            subprocess.run(  # nosec
                 cmd_pull, check=True, stdout=subprocess.PIPE, stderr=subprocess.STDOUT
             )
         except subprocess.CalledProcessError as err:
@@ -61,7 +61,7 @@ def save_docker_image(self) -> None:
             os.path.join(self.save_directory, self.get_image_name()),
             self.req,
         ]
-        subprocess.run(cmd_save, check=True)
+        subprocess.run(cmd_save, check=True)  # nosec
         _LOGGER.info(
             f"Image successfully pulled: {self.save_directory}/{self.get_image_name()}"
         )
@@ -79,7 +79,7 @@ class SingularityImagePuller(ImagePuller):
     def __init__(self, req: str, save_directory: str) -> None:
         """Create a Singularity-based software container image downloader."""
         super(SingularityImagePuller, self).__init__(req, save_directory)
-        version = subprocess.check_output(
+        version = subprocess.check_output(  # nosec
             ["singularity", "--version"], universal_newlines=True
         )
         if version.startswith("singularity version "):