[release-2.25] pre-commit: make hooks self contained + ci config (#11359)

* Use alternate self-sufficient shellcheck precommit

This pre-commit does not require prerequisite on the host, making it
easier to run in CI workflows.

* Switch to upstream ansible-lint pre-commit hook

This way, the hook is self contained and does not depend on a previous
virtualenv installation.

* pre-commit: fix hooks dependencies

- ansible-syntax-check
- tox-inventory-builder
- jinja-syntax-check

* Fix ci-matrix pre-commit hook

- Remove dependency of pydblite which fails to setup on recent pythons
- Discard shell script and put everything into pre-commit

* pre-commit: apply autofixes hooks and fix the rest manually

- markdownlint (manual fix)
- end-of-file-fixer
- requirements-txt-fixer
- trailing-whitespace

* Convert check_typo to pre-commit + use maintained version

client9/misspell is unmaintained, and has been forked by the golangci
team, see https://github.com/client9/misspell/issues/197#issuecomment-1596318684.

They haven't yet added a pre-commit config, so use my fork with the
pre-commit hook config until the pull request is merged.

* collection-build-install convert to pre-commit

* Run pre-commit hooks in dynamic pipeline

Use gitlab dynamic child pipelines feature to have one source of truth
for the pre-commit jobs, the pre-commit config file.

Use one cache per pre-commit. This should reduce the "fetching cache"
time steps in gitlab-ci, since each job will have a separate cache with
only its hook installed.

* Remove gitlab-ci job done in pre-commit

* pre-commit: adjust mardownlint default, md fixes

Use a style file as recommended by upstream. This makes for only one
source of truth.
Conserve previous upstream default for MD007 (upstream default changed
here https://github.com/markdownlint/markdownlint/pull/373)

* Update pre-commit hooks

---------

Co-authored-by: Max Gautier <mg@max.gautier.name>

tests/scripts/check_typo.sh

@@ -1,12 +0,0 @@
-#!/bin/bash
-
-# cd to the root directory of kubespray
-cd $(dirname $0)/../../
-
-rm ./misspell*
-
-set -e
-wget https://github.com/client9/misspell/releases/download/v0.3.4/misspell_0.3.4_linux_64bit.tar.gz
-tar -zxvf ./misspell_0.3.4_linux_64bit.tar.gz
-chmod 755 ./misspell
-git ls-files | grep -v OWNERS_ALIASES | xargs ./misspell -error

tests/scripts/collection-build-install.sh

@@ -0,0 +1,7 @@
+#!/bin/sh -e
+export ANSIBLE_COLLECTIONS_PATH="./ansible_collections"
+ansible-galaxy collection build --force
+ansible-galaxy collection install kubernetes_sigs-kubespray-$(grep "^version:" galaxy.yml | awk '{print $2}').tar.gz
+ansible-galaxy collection list $(egrep -i '(name:\s+|namespace:\s+)' galaxy.yml | awk '{print $2}' | tr '\n' '.' | sed 's|\.$||g') | grep "^kubernetes_sigs.kubespray"
+test -f ansible_collections/kubernetes_sigs/kubespray/playbooks/cluster.yml
+test -f ansible_collections/kubernetes_sigs/kubespray/playbooks/reset.yml

tests/scripts/md-table/main.py

@@ -4,7 +4,6 @@ import sys
 import glob
 from pathlib import Path
 import yaml
-from pydblite import Base
 import re
 import jinja2
 import sys
@@ -14,6 +13,7 @@ from pprint import pprint
 
 parser = argparse.ArgumentParser(description='Generate a Markdown table representing the CI test coverage')
 parser.add_argument('--dir', default='tests/files/', help='folder with test yml files')
+parser.add_argument('--output', default='docs/developers/ci.md', help='output file')
 
 
 args = parser.parse_args()
@@ -24,25 +24,26 @@ env = jinja2.Environment(loader=jinja2.FileSystemLoader(searchpath=sys.path[0]))
 # Data represents CI coverage data matrix
 class Data:
     def __init__(self):
-        self.db = Base(':memory:')
-        self.db.create('container_manager', 'network_plugin', 'operating_system')
+        self.container_managers = set()
+        self.network_plugins = set()
+        self.os = set()
+        self.combination = set()
 
 
-    def set(self, container_manager, network_plugin, operating_system):
-        self.db.insert(container_manager=container_manager, network_plugin=network_plugin, operating_system=operating_system)
-        self.db.commit()
-    def exists(self, container_manager, network_plugin, operating_system):
-        return len((self.db("container_manager") == container_manager) & (self.db("network_plugin") == network_plugin) & (self.db("operating_system") == operating_system)) > 0
+    def set(self, container_manager, network_plugin, os):
+        self.container_managers.add(container_manager)
+        self.network_plugins.add(network_plugin)
+        self.os.add(os)
+        self.combination.add(container_manager+network_plugin+os)
+
+    def exists(self, container_manager, network_plugin, os):
+        return (container_manager+network_plugin+os) in self.combination
 
     def jinja(self):
         template = env.get_template('table.md.j2')
-        container_engines = list(self.db.get_unique_ids('container_manager'))
-        network_plugins = list(self.db.get_unique_ids("network_plugin"))
-        operating_systems = list(self.db.get_unique_ids("operating_system"))
-
-        container_engines.sort()
-        network_plugins.sort()
-        operating_systems.sort()
+        container_engines = sorted(self.container_managers)
+        network_plugins = sorted(self.network_plugins)
+        operating_systems = sorted(self.os)
 
         return template.render(
             container_engines=container_engines,
@@ -91,6 +92,5 @@ for f in files:
     network_plugin = y.get('kube_network_plugin', 'calico')
     x = re.match(r"^[a-z-]+_([a-z0-9]+).*", f.name)
     operating_system = x.group(1)
-    data.set(container_manager=container_manager, network_plugin=network_plugin, operating_system=operating_system)
-#print(data.markdown())
-print(data.jinja())
+    data.set(container_manager=container_manager, network_plugin=network_plugin, os=operating_system)
+print(data.jinja(), file=open(args.output, 'w'))

tests/scripts/md-table/requirements.txt

@@ -1,4 +0,0 @@
-jinja2
-pathlib ; python_version < '3.10'
-pyaml
-pydblite

tests/scripts/md-table/test.sh

@@ -1,11 +0,0 @@
-#!/bin/bash
-set -euxo pipefail
-
-echo "Install requirements..."
-pip install -r ./tests/scripts/md-table/requirements.txt
-
-echo "Generate current file..."
-./tests/scripts/md-table/main.py > tmp.md
-
-echo "Compare docs/developers/ci.md with actual tests in tests/files/*.yml ..."
-cmp docs/developers/ci.md tmp.md