Squashed 'release-tools/' changes from 3b6d17b1..5489de6

.prow.sh

 #! /bin/bash -e
+
+# Copyright 2021 The Kubernetes Authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
 #
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
 # This is for testing csi-release-tools itself in Prow. All other
 # repos use prow.sh for that, but as csi-release-tools isn't a normal
 # repo with some Go code in it, it has a custom Prow test script.
 
 ./verify-shellcheck.sh "$(pwd)"
+./verify-spelling.sh "$(pwd)"
+./verify-boilerplate.sh "$(pwd)"

KUBERNETES_CSI_OWNERS_ALIASES

+# See the OWNERS docs: https://git.k8s.io/community/contributors/guide/owners.md
+
+aliases:
+
+  # SIG-Storage chairs and leads should always have approval rights in all repos.
+  # Others may be added as needed here or in each repo.
+  kubernetes-csi-approvers:
+  - jsafrane
+  - msau42
+  - saad-ali
+  - xing-yang
+
+  # Reviewers are automatically assigned to new PRs. The following
+  # reviewers will be active in all repos. Other reviewers can be
+  # added in each repo.
+  #
+  # Reviewers are encouraged to set the "Busy" flag in their GitHub status
+  # when they are temporarily unable to review PRs.
+  kubernetes-csi-reviewers:
+  - andyzhangx
+  - chrishenzie
+  - ggriffiths
+  - gnufied
+  - humblec
+  - j-griffith
+  - Jiawei0227
+  - jingxu97
+  - jsafrane
+  - pohly
+  - xing-yang
+
+# This documents who previously contributed to Kubernetes-CSI
+# as approver.
+emeritus_approver:
+- lpabon
+- sbezverk
+- vladimirvivien
+
+# This documents who previously contributed to Kubernetes-CSI
+# as reviewer.
+emeritus_reviewer:
+- lpabon
+- saad-ali
+- sbezverk
+- vladimirvivien

OWNERS

 # See the OWNERS docs: https://git.k8s.io/community/contributors/guide/owners.md
 
 approvers:
-- saad-ali
-- msau42
+- kubernetes-csi-approvers
 - pohly
 
 reviewers:
-- saad-ali
-- msau42
-- pohly
+- kubernetes-csi-reviewers

OWNERS_ALIASES

+KUBERNETES_CSI_OWNERS_ALIASES
\ No newline at end of file

README.md

@@ -21,7 +21,11 @@ The expected repository layout is:
    Dockerfile in the root when only building a single command
  - `Makefile` - includes `release-tools/build.make` and sets
    configuration variables
- - `.travis.yml` - a symlink to `release-tools/.travis.yml`
+ - `.prow.sh` script which imports `release-tools/prow.sh`
+   and may contain further customization
+ - `.cloudbuild.sh` and `cloudbuild.yaml` as symlinks to
+   the corresponding files in `release-tools` or (if necessary)
+   as custom files
 
 To create a release, tag a certain revision with a name that
 starts with `v`, for example `v1.0.0`, then `make push`
@@ -38,16 +42,23 @@ images. Building from master creates the main `canary` image.
 Sharing and updating
 --------------------
 
-[`git subtree`](https://github.com/git/git/blob/master/contrib/subtree/git-subtree.txt)
+[`git subtree`](https://github.com/git/git/blob/HEAD/contrib/subtree/git-subtree.txt)
 is the recommended way of maintaining a copy of the rules inside the
 `release-tools` directory of a project. This way, it is possible to make
 changes also locally, test them and then push them back to the shared
 repository at a later time.
 
+We no longer care about importing the full commit history, so `--squash` should be used
+when submitting a `release-tools` update. Also make sure that the PR for that
+contains the automatically generated commit message in the PR description.
+It contains the list of individual commits that were squashed. The script from
+https://github.com/kubernetes-csi/csi-release-tools/issues/7 can create such
+PRs automatically.
+
 Cheat sheet:
 
-- `git subtree add --prefix=release-tools https://github.com/kubernetes-csi/csi-release-tools.git master` - add release tools to a repo which does not have them yet (only once)
-- `git subtree pull --prefix=release-tools https://github.com/kubernetes-csi/csi-release-tools.git master` - update local copy to latest upstream (whenever upstream changes)
+- `git subtree add --squash --prefix=release-tools https://github.com/kubernetes-csi/csi-release-tools.git master` - add release tools to a repo which does not have them yet (only once)
+- `git subtree pull --squash --prefix=release-tools https://github.com/kubernetes-csi/csi-release-tools.git master` - update local copy to latest upstream (whenever upstream changes)
 - edit, `git commit`, `git subtree push --prefix=release-tools git@github.com:<user>/csi-release-tools.git <my-new-or-existing-branch>` - push to a new branch before submitting a PR
 
 verify-shellcheck.sh
@@ -78,7 +89,7 @@ main
 
 All Kubernetes-CSI repos are expected to switch to Prow. For details
 on what is enabled in Prow, see
-https://github.com/kubernetes/test-infra/tree/master/config/jobs/kubernetes-csi
+https://github.com/kubernetes/test-infra/tree/HEAD/config/jobs/kubernetes-csi
 
 Test results for periodic jobs are visible in
 https://testgrid.k8s.io/sig-storage-csi-ci

SECURITY_CONTACTS

@@ -4,7 +4,7 @@
 # to for triaging and handling of incoming issues.
 #
 # The below names agree to abide by the
-# [Embargo Policy](https://github.com/kubernetes/sig-release/blob/master/security-release-process-documentation/security-release-process.md#embargo-policy)
+# [Embargo Policy](https://github.com/kubernetes/sig-release/blob/HEAD/security-release-process-documentation/security-release-process.md#embargo-policy)
 # and will be removed and replaced if they violate that agreement.
 #
 # DO NOT REPORT SECURITY VULNERABILITIES DIRECTLY TO THESE NAMES, FOLLOW THE

SIDECAR_RELEASE_PROCESS.md

@@ -9,13 +9,8 @@ The release manager must:
 * Be a member of the kubernetes-csi organization. Open an
   [issue](https://github.com/kubernetes/org/issues/new?assignees=&labels=area%2Fgithub-membership&template=membership.md&title=REQUEST%3A+New+membership+for+%3Cyour-GH-handle%3E) in
   kubernetes/org to request membership
-* Be a top level approver for the repository. To become a top level approver,
-  the candidate must demonstrate ownership and deep knowledge of the repository
-  through active maintainence, responding to and fixing issues, reviewing PRs,
-  test triage.
-* Be part of the maintainers or admin group for the repository. admin is a
-  superset of maintainers, only maintainers level is required for cutting a
-  release.  Membership can be requested by submitting a PR to kubernetes/org.
+* Be part of the maintainers group for the repository.
+  Membership can be requested by submitting a PR to kubernetes/org.
   [Example](https://github.com/kubernetes/org/pull/1467)
 
 ## Updating CI Jobs
@@ -31,16 +26,16 @@ naming convention `<hostpath-deployment-version>-on-<kubernetes-version>`.
 1. "-on-master" jobs are the closest reflection to the new Kubernetes version.
 1. Fixes to our prow.sh CI script can be tested in the [CSI hostpath
    repo](https://github.com/kubernetes-csi/csi-driver-host-path) by modifying
-   [prow.sh](https://github.com/kubernetes-csi/csi-driver-host-path/blob/master/release-tools/prow.sh)
+   [prow.sh](https://github.com/kubernetes-csi/csi-driver-host-path/blob/HEAD/release-tools/prow.sh)
    along with any overrides in
-   [.prow.sh](https://github.com/kubernetes-csi/csi-driver-host-path/blob/master/.prow.sh)
+   [.prow.sh](https://github.com/kubernetes-csi/csi-driver-host-path/blob/HEAD/.prow.sh)
    to mirror the failing environment. Once e2e tests are passing (verify-unit tests
    will fail), then the prow.sh changes can be submitted to [csi-release-tools](https://github.com/kubernetes-csi/csi-release-tools).
 1. Changes can then be updated in all the sidecar repos and hostpath driver repo
    by following the [update
-   instructions](https://github.com/kubernetes-csi/csi-release-tools/blob/master/README.md#sharing-and-updating).
+   instructions](https://github.com/kubernetes-csi/csi-release-tools/blob/HEAD/README.md#sharing-and-updating).
 1. New pull and CI jobs are configured by adding new K8s versions to the top of
-   [gen-jobs.sh](https://github.com/kubernetes/test-infra/blob/master/config/jobs/kubernetes-csi/gen-jobs.sh).
+   [gen-jobs.sh](https://github.com/kubernetes/test-infra/blob/HEAD/config/jobs/kubernetes-csi/gen-jobs.sh).
    New pull jobs that have been unverified should be initially made optional by
    setting the new K8s version as
    [experimental](https://github.com/kubernetes/test-infra/blob/a1858f46d6014480b130789df58b230a49203a64/config/jobs/kubernetes-csi/gen-jobs.sh#L40).
@@ -52,7 +47,7 @@ naming convention `<hostpath-deployment-version>-on-<kubernetes-version>`.
 1. Identify all issues and ongoing PRs that should go into the release, and
   drive them to resolution.
 1. Download v2.8+ [K8s release notes
-  generator](https://github.com/kubernetes/release/tree/master/cmd/release-notes)
+  generator](https://github.com/kubernetes/release/tree/HEAD/cmd/release-notes)
 1. Generate release notes for the release. Replace arguments with the relevant
   information.
     * Clean up old cached information (also needed if you are generating release
@@ -95,12 +90,56 @@ naming convention `<hostpath-deployment-version>-on-<kubernetes-version>`.
 1. Check [image build status](https://k8s-testgrid.appspot.com/sig-storage-image-build).
 1. Promote images from k8s-staging-sig-storage to k8s.gcr.io/sig-storage. From
    the [k8s image
-   repo](https://github.com/kubernetes/k8s.io/tree/master/k8s.gcr.io/images/k8s-staging-sig-storage),
+   repo](https://github.com/kubernetes/k8s.io/tree/HEAD/k8s.gcr.io/images/k8s-staging-sig-storage),
    run `./generate.sh > images.yaml`, and send a PR with the updated images.
    Once merged, the image promoter will copy the images from staging to prod.
 1. Update [kubernetes-csi/docs](https://github.com/kubernetes-csi/docs) sidecar
    and feature pages with the new released version.
 1. After all the sidecars have been released, update
-   CSI hostpath driver with the new sidecars in the [CSI repo](https://github.com/kubernetes-csi/csi-driver-host-path/tree/master/deploy)
+   CSI hostpath driver with the new sidecars in the [CSI repo](https://github.com/kubernetes-csi/csi-driver-host-path/tree/HEAD/deploy)
    and [k/k
-   in-tree](https://github.com/kubernetes/kubernetes/tree/master/test/e2e/testing-manifests/storage-csi/hostpath/hostpath)
+   in-tree](https://github.com/kubernetes/kubernetes/tree/HEAD/test/e2e/testing-manifests/storage-csi/hostpath/hostpath)
+
+## Adding support for a new Kubernetes release
+
+1. Add the new release to `k8s_versions` in
+   https://github.com/kubernetes/test-infra/blob/090dec5dd535d5f61b7ba52e671a810f5fc13dfd/config/jobs/kubernetes-csi/gen-jobs.sh#L25
+   to enable generating a job for it. Set `experimental_k8s_version`
+   in
+   https://github.com/kubernetes/test-infra/blob/090dec5dd535d5f61b7ba52e671a810f5fc13dfd/config/jobs/kubernetes-csi/gen-jobs.sh#L40
+   to ensure that the new jobs aren't run for PRs unless explicitly
+   requested. Generate and submit the new jobs.
+1. Create a test PR to try out the new job in some repo with `/test
+   pull-kubernetes-csi-<repo>-<x.y>-on-kubernetes-<x.y>` where x.y
+   matches the Kubernetes release. Alternatively, run .prow.sh in that
+   repo locally with `CSI_PROW_KUBERNETES_VERSION=x.y.z`.
+1. Optional: update to a [new
+   release](https://github.com/kubernetes-sigs/kind/tags) of kind with
+   pre-built images for the new Kubernetes release. This is optional
+   if the current version of kind is able to build images for the new
+   Kubernetes release. However, jobs require less resources when they
+   don't need to build those images from the Kubernetes source code.
+   This change needs to be tried out in a PR against a component
+   first, then get submitted against csi-release-tools.
+1. Optional: propagate the updated csi-release-tools to all components
+   with the script from
+   https://github.com/kubernetes-csi/csi-release-tools/issues/7#issuecomment-707025402
+1. Once it is likely to work in all components, unset
+   `experimental_k8s_version` and submit the updated jobs.
+1. Once all sidecars for the new Kubernetes release are released,
+   either bump the version number of the images in the existing
+   [csi-driver-host-path
+   deployments](https://github.com/kubernetes-csi/csi-driver-host-path/tree/HEAD/deploy)
+   and/or create a new deployment, depending on what Kubernetes
+   release an updated sidecar is compatible with. If no new deployment
+   is needed, then add a symlink to document that there intentionally
+   isn't a separate deployment. This symlink is not needed for Prow
+   testing because that will use "kubernetes-latest" as fallback.
+   Update that link when creating a new deployment.
+1. Create a new csi-driver-host-path release.
+1. Bump `CSI_PROW_DRIVER_VERSION` in prow.sh to that new release and
+   (eventually) roll that change out to all repos by updating
+   `release-tools` in them. This is used when testing manually. The
+   Prow jobs override that value, so also update
+   `hostpath_driver_version` in
+   https://github.com/kubernetes/test-infra/blob/91b04e6af3a40a9bcff25aa030850a4721e2dd2b/config/jobs/kubernetes-csi/gen-jobs.sh#L46-L47

boilerplate/boilerplate.Dockerfile.txt

+# Copyright YEAR The Kubernetes Authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.

boilerplate/boilerplate.Makefile.txt

+# Copyright YEAR The Kubernetes Authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
\ No newline at end of file

boilerplate/boilerplate.bzl.txt

+# Copyright YEAR The Kubernetes Authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
\ No newline at end of file

boilerplate/boilerplate.go.txt

+/*
+Copyright YEAR The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
\ No newline at end of file

boilerplate/boilerplate.py

+#!/usr/bin/env python
+
+# Copyright 2019 The Kubernetes Authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from __future__ import print_function
+
+import argparse
+import difflib
+import glob
+import os
+import re
+import sys
+from datetime import date
+
+parser = argparse.ArgumentParser()
+parser.add_argument(
+    "filenames",
+    help="list of files to check, all files if unspecified",
+    nargs='*')
+
+# Rootdir defaults to the directory **above** the repo-infra dir.
+rootdir = os.path.dirname(__file__) + "./../../"
+rootdir = os.path.abspath(rootdir)
+parser.add_argument(
+    "--rootdir", default=rootdir, help="root directory to examine")
+
+default_boilerplate_dir = os.path.abspath(os.path.dirname(__file__))
+
+parser.add_argument(
+    "--boilerplate-dir", default=default_boilerplate_dir)
+
+parser.add_argument(
+    "-v", "--verbose",
+    help="give verbose output regarding why a file does not pass",
+    action="store_true")
+
+args = parser.parse_args()
+
+verbose_out = sys.stderr if args.verbose else open("/dev/null", "w")
+
+def get_refs():
+    refs = {}
+
+    for path in glob.glob(os.path.join(args.boilerplate_dir, "boilerplate.*.txt")):
+        extension = os.path.basename(path).split(".")[1]
+
+        ref_file = open(path, 'r')
+        ref = ref_file.read().splitlines()
+        ref_file.close()
+        refs[extension] = ref
+
+    return refs
+
+def file_passes(filename, refs, regexs):
+    try:
+        f = open(filename, 'r')
+    except Exception as exc:
+        print("Unable to open %s: %s" % (filename, exc), file=verbose_out)
+        return False
+
+    data = f.read()
+    f.close()
+
+    basename = os.path.basename(filename)
+    extension = file_extension(filename)
+    if extension != "":
+        ref = refs[extension]
+    else:
+        ref = refs[basename]
+
+    # remove build tags from the top of Go files
+    if extension == "go":
+        p = regexs["go_build_constraints"]
+        (data, found) = p.subn("", data, 1)
+
+    # remove shebang from the top of shell files
+    if extension == "sh" or extension == "py":
+        p = regexs["shebang"]
+        (data, found) = p.subn("", data, 1)
+
+    data = data.splitlines()
+
+    # if our test file is smaller than the reference it surely fails!
+    if len(ref) > len(data):
+        print('File %s smaller than reference (%d < %d)' %
+              (filename, len(data), len(ref)),
+              file=verbose_out)
+        return False
+
+    # trim our file to the same number of lines as the reference file
+    data = data[:len(ref)]
+
+    p = regexs["year"]
+    for d in data:
+        if p.search(d):
+            print('File %s is missing the year' % filename, file=verbose_out)
+            return False
+
+    # Replace all occurrences of the regex "CURRENT_YEAR|...|2016|2015|2014" with "YEAR"
+    p = regexs["date"]
+    for i, d in enumerate(data):
+        (data[i], found) = p.subn('YEAR', d)
+        if found != 0:
+            break
+
+    # if we don't match the reference at this point, fail
+    if ref != data:
+        print("Header in %s does not match reference, diff:" % filename, file=verbose_out)
+        if args.verbose:
+            print(file=verbose_out)
+            for line in difflib.unified_diff(ref, data, 'reference', filename, lineterm=''):
+                print(line, file=verbose_out)
+            print(file=verbose_out)
+        return False
+
+    return True
+
+def file_extension(filename):
+    return os.path.splitext(filename)[1].split(".")[-1].lower()
+
+skipped_dirs = ['Godeps', 'third_party', '_gopath', '_output', '.git',
+                'cluster/env.sh', 'vendor', 'test/e2e/generated/bindata.go',
+                'repo-infra/verify/boilerplate/test', '.glide']
+
+def normalize_files(files):
+    newfiles = []
+    for pathname in files:
+        if any(x in pathname for x in skipped_dirs):
+            continue
+        newfiles.append(pathname)
+    return newfiles
+
+def get_files(extensions):
+    files = []
+    if len(args.filenames) > 0:
+        files = args.filenames
+    else:
+        for root, dirs, walkfiles in os.walk(args.rootdir):
+            # don't visit certain dirs. This is just a performance improvement
+            # as we would prune these later in normalize_files(). But doing it
+            # cuts down the amount of filesystem walking we do and cuts down
+            # the size of the file list
+            for d in skipped_dirs:
+                if d in dirs:
+                    dirs.remove(d)
+
+            for name in walkfiles:
+                pathname = os.path.join(root, name)
+                files.append(pathname)
+
+    files = normalize_files(files)
+
+    outfiles = []
+    for pathname in files:
+        basename = os.path.basename(pathname)
+        extension = file_extension(pathname)
+        if extension in extensions or basename in extensions:
+            outfiles.append(pathname)
+    return outfiles
+
+def get_regexs():
+    regexs = {}
+    # Search for "YEAR" which exists in the boilerplate, but shouldn't in the real thing
+    regexs["year"] = re.compile( 'YEAR' )
+    # dates can be 2014, 2015, 2016, ..., CURRENT_YEAR, company holder names can be anything
+    years = range(2014, date.today().year + 1)
+    regexs["date"] = re.compile( '(%s)' % "|".join(map(lambda l: str(l), years)) )
+    # strip // +build \n\n build constraints
+    regexs["go_build_constraints"] = re.compile(r"^(// \+build.*\n)+\n", re.MULTILINE)
+    # strip #!.* from shell scripts
+    regexs["shebang"] = re.compile(r"^(#!.*\n)\n*", re.MULTILINE)
+    return regexs
+
+
+
+def main():
+    regexs = get_regexs()
+    refs = get_refs()
+    filenames = get_files(refs.keys())
+
+    for filename in filenames:
+        if not file_passes(filename, refs, regexs):
+            print(filename, file=sys.stdout)
+
+    return 0
+
+if __name__ == "__main__":
+    sys.exit(main())

boilerplate/boilerplate.py.txt

+# Copyright YEAR The Kubernetes Authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.

boilerplate/boilerplate.sh.txt

+# Copyright YEAR The Kubernetes Authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
\ No newline at end of file

build.make

@@ -12,12 +12,19 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+# force the usage of /bin/bash instead of /bin/sh
+SHELL := /bin/bash
+
 .PHONY: build-% build container-% container push-% push clean test
 
 # A space-separated list of all commands in the repository, must be
 # set in main Makefile of a repository.
 # CMDS=
 
+# Normally, commands are expected in "cmd". That can be changed for a
+# repository to something else by setting CMDS_DIR before including build.make.
+CMDS_DIR ?= cmd
+
 # This is the default. It can be overridden in the main Makefile after
 # including build.make.
 REGISTRY_NAME?=quay.io/k8scsi
@@ -63,30 +70,39 @@ endif
 # Specific packages can be excluded from each of the tests below by setting the *_FILTER_CMD variables
 # to something like "| grep -v 'github.com/kubernetes-csi/project/pkg/foobar'". See usage below.
 
-# BUILD_PLATFORMS contains a set of <os> <arch> <suffix> triplets,
+# BUILD_PLATFORMS contains a set of tuples [os arch suffix base_image addon_image]
 # separated by semicolon. An empty variable or empty entry (= just a
 # semicolon) builds for the default platform of the current Go
 # toolchain.
 BUILD_PLATFORMS =
 
 # Add go ldflags using LDFLAGS at the time of compilation.
-IMPORTPATH_LDFLAGS = -X main.version=$(REV) 
+IMPORTPATH_LDFLAGS = -X main.version=$(REV)
 EXT_LDFLAGS = -extldflags "-static"
-LDFLAGS = 
+LDFLAGS =
 FULL_LDFLAGS = $(LDFLAGS) $(IMPORTPATH_LDFLAGS) $(EXT_LDFLAGS)
 # This builds each command (= the sub-directories of ./cmd) for the target platform(s)
 # defined by BUILD_PLATFORMS.
 $(CMDS:%=build-%): build-%: check-go-version-go
 	mkdir -p bin
-	echo '$(BUILD_PLATFORMS)' | tr ';' '\n' | while read -r os arch suffix; do \
-		if ! (set -x; CGO_ENABLED=0 GOOS="$$os" GOARCH="$$arch" go build $(GOFLAGS_VENDOR) -a -ldflags '$(FULL_LDFLAGS)' -o "./bin/$*$$suffix" ./cmd/$*); then \
+	# os_arch_seen captures all of the $$os-$$arch seen for the current binary
+	# that we want to build, if we've seen an $$os-$$arch before it means that
+	# we don't need to build it again, this is done to avoid building
+	# the windows binary multiple times (see the default value of $$BUILD_PLATFORMS)
+	export os_arch_seen="" && echo '$(BUILD_PLATFORMS)' | tr ';' '\n' | while read -r os arch suffix base_image addon_image; do \
+		os_arch_seen_pre=$${os_arch_seen%%$$os-$$arch*}; \
+		if ! [ $${#os_arch_seen_pre} = $${#os_arch_seen} ]; then \
+			continue; \
+		fi; \
+		if ! (set -x; cd ./$(CMDS_DIR)/$* && CGO_ENABLED=0 GOOS="$$os" GOARCH="$$arch" go build $(GOFLAGS_VENDOR) -a -ldflags '$(FULL_LDFLAGS)' -o "$(abspath ./bin)/$*$$suffix" .); then \
 			echo "Building $* for GOOS=$$os GOARCH=$$arch failed, see error(s) above."; \
 			exit 1; \
 		fi; \
+		os_arch_seen+=";$$os-$$arch"; \
 	done
 
 $(CMDS:%=container-%): container-%: build-%
-	docker build -t $*:latest -f $(shell if [ -e ./cmd/$*/Dockerfile ]; then echo ./cmd/$*/Dockerfile; else echo Dockerfile; fi) --label revision=$(REV) .
+	docker build -t $*:latest -f $(shell if [ -e ./$(CMDS_DIR)/$*/Dockerfile ]; then echo ./$(CMDS_DIR)/$*/Dockerfile; else echo Dockerfile; fi) --label revision=$(REV) .
 
 $(CMDS:%=push-%): push-%: container-%
 	set -ex; \
@@ -121,7 +137,7 @@ DOCKER_BUILDX_CREATE_ARGS ?=
 # This target builds a multiarch image for one command using Moby BuildKit builder toolkit.
 # Docker Buildx is included in Docker 19.03.
 #
-# ./cmd/<command>/Dockerfile[.Windows] is used if found, otherwise Dockerfile[.Windows].
+# ./$(CMDS_DIR)/<command>/Dockerfile[.Windows] is used if found, otherwise Dockerfile[.Windows].
 # It is currently optional: if no such file exists, Windows images are not included,
 # even when Windows is listed in BUILD_PLATFORMS. That way, projects can test that
 # Windows binaries can be built before adding a Dockerfile for it.
@@ -131,29 +147,46 @@ DOCKER_BUILDX_CREATE_ARGS ?=
 # the tag for the resulting multiarch image.
 $(CMDS:%=push-multiarch-%): push-multiarch-%: check-pull-base-ref build-%
 	set -ex; \
-	DOCKER_CLI_EXPERIMENTAL=enabled; \
-	export DOCKER_CLI_EXPERIMENTAL; \
+	export DOCKER_CLI_EXPERIMENTAL=enabled; \
 	docker buildx create $(DOCKER_BUILDX_CREATE_ARGS) --use --name multiarchimage-buildertest; \
 	trap "docker buildx rm multiarchimage-buildertest" EXIT; \
-	dockerfile_linux=$$(if [ -e ./cmd/$*/Dockerfile ]; then echo ./cmd/$*/Dockerfile; else echo Dockerfile; fi); \
-	dockerfile_windows=$$(if [ -e ./cmd/$*/Dockerfile.Windows ]; then echo ./cmd/$*/Dockerfile.Windows; else echo Dockerfile.Windows; fi); \
+	dockerfile_linux=$$(if [ -e ./$(CMDS_DIR)/$*/Dockerfile ]; then echo ./$(CMDS_DIR)/$*/Dockerfile; else echo Dockerfile; fi); \
+	dockerfile_windows=$$(if [ -e ./$(CMDS_DIR)/$*/Dockerfile.Windows ]; then echo ./$(CMDS_DIR)/$*/Dockerfile.Windows; else echo Dockerfile.Windows; fi); \
 	if [ '$(BUILD_PLATFORMS)' ]; then build_platforms='$(BUILD_PLATFORMS)'; else build_platforms="linux amd64"; fi; \
 	if ! [ -f "$$dockerfile_windows" ]; then \
-		build_platforms="$$(echo "$$build_platforms" | sed -e 's/windows *[^ ]* *.exe//g' -e 's/; *;/;/g')"; \
+		build_platforms="$$(echo "$$build_platforms" | sed -e 's/windows *[^ ]* *.exe *[^ ]* *[^ ]*//g' -e 's/; *;/;/g' -e 's/;[ ]*$$//')"; \
 	fi; \
 	pushMultiArch () { \
 		tag=$$1; \
-		echo "$$build_platforms" | tr ';' '\n' | while read -r os arch suffix; do \
+		echo "$$build_platforms" | tr ';' '\n' | while read -r os arch suffix base_image addon_image; do \
+			escaped_base_image=$${base_image/:/-}; \
+			if ! [ -z $$escaped_base_image ]; then escaped_base_image+="-"; fi; \
 			docker buildx build --push \
-				--tag $(IMAGE_NAME):$$arch-$$os-$$tag \
+				--tag $(IMAGE_NAME):$$arch-$$os-$$escaped_base_image$$tag \
 				--platform=$$os/$$arch \
 				--file $$(eval echo \$${dockerfile_$$os}) \
 				--build-arg binary=./bin/$*$$suffix \
+				--build-arg ARCH=$$arch \
+				--build-arg BASE_IMAGE=$$base_image \
+				--build-arg ADDON_IMAGE=$$addon_image \
 				--label revision=$(REV) \
 				.; \
 		done; \
-		images=$$(echo "$$build_platforms" | tr ';' '\n' | while read -r os arch suffix; do echo $(IMAGE_NAME):$$arch-$$os-$$tag; done); \
+		images=$$(echo "$$build_platforms" | tr ';' '\n' | while read -r os arch suffix base_image addon_image; do \
+			escaped_base_image=$${base_image/:/-}; \
+			if ! [ -z $$escaped_base_image ]; then escaped_base_image+="-"; fi; \
+			echo $(IMAGE_NAME):$$arch-$$os-$$escaped_base_image$$tag; \
+		done); \
 		docker manifest create --amend $(IMAGE_NAME):$$tag $$images; \
+		echo "$$build_platforms" | tr ';' '\n' | while read -r os arch suffix base_image addon_image; do \
+			if [ $$os = "windows" ]; then \
+				escaped_base_image=$${base_image/:/-}; \
+				if ! [ -z $$escaped_base_image ]; then escaped_base_image+="-"; fi; \
+				image=$(IMAGE_NAME):$$arch-$$os-$$escaped_base_image$$tag; \
+				os_version=$$(docker manifest inspect mcr.microsoft.com/windows/$${base_image} | grep "os.version" | head -n 1 | awk '{print $$2}' | sed -e 's/"//g') || true; \
+				docker manifest annotate --os-version $$os_version $(IMAGE_NAME):$$tag $$image; \
+			fi; \
+		done; \
 		docker manifest push -p $(IMAGE_NAME):$$tag; \
 	}; \
 	if [ $(PULL_BASE_REF) = "master" ]; then \
@@ -275,3 +308,15 @@ test-shellcheck:
 .PHONY: check-go-version-%
 check-go-version-%:
 	./release-tools/verify-go-version.sh "$*"
+
+# Test for spelling errors.
+.PHONY: test-spelling
+test-spelling:
+	@ echo; echo "### $@:"
+	@ ./release-tools/verify-spelling.sh "$(pwd)"
+
+# Test the boilerplates of the files.
+.PHONY: test-boilerplate
+test-boilerplate:
+	@ echo; echo "### $@:"
+	@ ./release-tools/verify-boilerplate.sh "$(pwd)"

cloudbuild.sh

 #! /bin/bash
 
+# Copyright 2021 The Kubernetes Authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
 # shellcheck disable=SC1091
 . release-tools/prow.sh
 

cloudbuild.yaml

@@ -10,10 +10,10 @@
 #   because binaries will get built for different architectures and then
 #   get copied from the built host into the container image
 #
-# See https://github.com/kubernetes/test-infra/blob/master/config/jobs/image-pushing/README.md
+# See https://github.com/kubernetes/test-infra/blob/HEAD/config/jobs/image-pushing/README.md
 # for more details on image pushing process in Kubernetes.
 #
-# To promote release images, see https://github.com/kubernetes/k8s.io/tree/master/k8s.gcr.io/images/k8s-staging-sig-storage.
+# To promote release images, see https://github.com/kubernetes/k8s.io/tree/HEAD/k8s.gcr.io/images/k8s-staging-sig-storage.
 
 # This must be specified in seconds. If omitted, defaults to 600s (10 mins).
 # Building three images in external-snapshotter takes roughly half an hour,
@@ -25,9 +25,9 @@ options:
   substitution_option: ALLOW_LOOSE
 steps:
   # The image must contain bash and curl. Ideally it should also contain
-  # the desired version of Go (currently defined in release-tools/travis.yml),
+  # the desired version of Go (currently defined in release-tools/prow.sh),
   # but that just speeds up the build and is not required.
-  - name: 'gcr.io/k8s-testimages/gcb-docker-gcloud:v20200421-a2bf5f8'
+  - name: 'gcr.io/k8s-testimages/gcb-docker-gcloud:v20210331-c732583'
     entrypoint: ./.cloudbuild.sh
     env:
     - GIT_TAG=${_GIT_TAG}

go-get-kubernetes.sh

@@ -55,6 +55,12 @@ mods=$( (set -x; curl --silent --show-error --fail "https://raw.githubuserconten
           sed -n 's|.*k8s.io/\(.*\) => ./staging/src/k8s.io/.*|k8s.io/\1|p'
    ) || die "failed to determine Kubernetes staging modules"
 for mod in $mods; do
+    if ! (env GO111MODULE=on go mod graph) | grep "$mod@" > /dev/null; then
+        echo "Kubernetes module $mod is not used, skipping"
+        # Remove the module from go.mod "replace" that was added by an older version of this script.
+        (set -x;  env GO111MODULE=on go mod edit "-dropreplace=$mod") || die "'go mod edit' failed"
+        continue
+    fi
     # The presence of a potentially incomplete go.mod file affects this command,
     # so move elsewhere.
     modinfo=$(set -x; cd /; env GO111MODULE=on go mod download -json "$mod@kubernetes-${k8s}") ||

prow.sh

 #! /bin/bash
-#
+
 # Copyright 2019 The Kubernetes Authors.
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
@@ -33,8 +33,6 @@
 # The expected environment is:
 # - $GOPATH/src/<import path> for the repository that is to be tested,
 #   with PR branch merged (when testing a PR)
-# - optional: bazel installed (when testing against Kubernetes master),
-#   must be recent enough for Kubernetes master
 # - running on linux-amd64
 # - kind (https://github.com/kubernetes-sigs/kind) installed
 # - optional: Go already installed
@@ -65,7 +63,22 @@ get_versioned_variable () {
     echo "$value"
 }
 
-configvar CSI_PROW_BUILD_PLATFORMS "linux amd64; windows amd64 .exe; linux ppc64le -ppc64le; linux s390x -s390x; linux arm64 -arm64" "Go target platforms (= GOOS + GOARCH) and file suffix of the resulting binaries"
+# This takes a version string like CSI_PROW_KUBERNETES_VERSION and
+# maps it to the corresponding git tag, branch or commit.
+version_to_git () {
+    version="$1"
+    shift
+    case "$version" in
+        latest|master) echo "master";;
+        release-*) echo "$version";;
+        *) echo "v$version";;
+    esac
+}
+
+# the list of windows versions was matched from:
+# - https://hub.docker.com/_/microsoft-windows-nanoserver
+# - https://hub.docker.com/_/microsoft-windows-servercore
+configvar CSI_PROW_BUILD_PLATFORMS "linux amd64; linux ppc64le -ppc64le; linux s390x -s390x; linux arm -arm; linux arm64 -arm64; windows amd64 .exe nanoserver:1809 servercore:ltsc2019; windows amd64 .exe nanoserver:1909 servercore:1909; windows amd64 .exe nanoserver:2004 servercore:2004; windows amd64 .exe nanoserver:20H2 servercore:20H2; windows amd64 .exe nanoserver:ltsc2022 servercore:ltsc2022" "Go target platforms (= GOOS + GOARCH) and file suffix of the resulting binaries"
 
 # If we have a vendor directory, then use it. We must be careful to only
 # use this for "make" invocations inside the project's repo itself because
@@ -73,37 +86,12 @@ configvar CSI_PROW_BUILD_PLATFORMS "linux amd64; windows amd64 .exe; linux ppc64
 # which is disabled with GOFLAGS=-mod=vendor).
 configvar GOFLAGS_VENDOR "$( [ -d vendor ] && echo '-mod=vendor' )" "Go flags for using the vendor directory"
 
-# Go versions can be specified seperately for different tasks
-# If the pre-installed Go is missing or a different
-# version, the required version here will get installed
-# from https://golang.org/dl/.
-go_from_travis_yml () {
-    grep "^ *- go:" "${RELEASE_TOOLS_ROOT}/travis.yml" | sed -e 's/.*go: *//'
-}
-configvar CSI_PROW_GO_VERSION_BUILD "$(go_from_travis_yml)" "Go version for building the component" # depends on component's source code
+configvar CSI_PROW_GO_VERSION_BUILD "1.16" "Go version for building the component" # depends on component's source code
 configvar CSI_PROW_GO_VERSION_E2E "" "override Go version for building the Kubernetes E2E test suite" # normally doesn't need to be set, see install_e2e
 configvar CSI_PROW_GO_VERSION_SANITY "${CSI_PROW_GO_VERSION_BUILD}" "Go version for building the csi-sanity test suite" # depends on CSI_PROW_SANITY settings below
 configvar CSI_PROW_GO_VERSION_KIND "${CSI_PROW_GO_VERSION_BUILD}" "Go version for building 'kind'" # depends on CSI_PROW_KIND_VERSION below
 configvar CSI_PROW_GO_VERSION_GINKGO "${CSI_PROW_GO_VERSION_BUILD}" "Go version for building ginkgo" # depends on CSI_PROW_GINKGO_VERSION below
 
-# kind version to use. If the pre-installed version is different,
-# the desired version is downloaded from https://github.com/kubernetes-sigs/kind/releases
-# (if available), otherwise it is built from source.
-configvar CSI_PROW_KIND_VERSION "v0.9.0" "kind"
-
-# kind images to use. Must match the kind version.
-# The release notes of each kind release list the supported images.
-configvar CSI_PROW_KIND_IMAGES "kindest/node:v1.19.1@sha256:98cf5288864662e37115e362b23e4369c8c4a408f99cbc06e58ac30ddc721600
-kindest/node:v1.18.8@sha256:f4bcc97a0ad6e7abaf3f643d890add7efe6ee4ab90baeb374b4f41a4c95567eb
-kindest/node:v1.17.11@sha256:5240a7a2c34bf241afb54ac05669f8a46661912eab05705d660971eeb12f6555
-kindest/node:v1.16.15@sha256:a89c771f7de234e6547d43695c7ab047809ffc71a0c3b65aa54eda051c45ed20
-kindest/node:v1.15.12@sha256:d9b939055c1e852fe3d86955ee24976cab46cba518abcb8b13ba70917e6547a6
-kindest/node:v1.14.10@sha256:ce4355398a704fca68006f8a29f37aafb49f8fc2f64ede3ccd0d9198da910146
-kindest/node:v1.13.12@sha256:1c1a48c2bfcbae4d5f4fa4310b5ed10756facad0b7a2ca93c7a4b5bae5db29f5" "kind images"
-
-# Use kind node-image --type=bazel by default, but allow to disable that.
-configvar CSI_PROW_USE_BAZEL true "use Bazel during 'kind node-image' invocation"
-
 # ginkgo test runner version to use. If the pre-installed version is
 # different, the desired version is built from source.
 configvar CSI_PROW_GINKGO_VERSION v1.7.0 "Ginkgo"
@@ -140,10 +128,36 @@ configvar CSI_PROW_KUBERNETES_VERSION 1.17.0 "Kubernetes"
 # when a Prow job just defines the Kubernetes version.
 csi_prow_kubernetes_version_suffix="$(echo "${CSI_PROW_KUBERNETES_VERSION}" | tr . _ | tr '[:lower:]' '[:upper:]' | sed -e 's/^RELEASE-//' -e 's/\([0-9]*\)_\([0-9]*\).*/\1_\2/')"
 
-# Work directory. It has to allow running executables, therefore /tmp
-# is avoided. Cleaning up after the script is intentionally left to
-# the caller.
-configvar CSI_PROW_WORK "$(mkdir -p "$GOPATH/pkg" && mktemp -d "$GOPATH/pkg/csiprow.XXXXXXXXXX")" "work directory"
+# Only the latest KinD is (eventually) guaranteed to work with the
+# latest Kubernetes. For example, KinD 0.10.0 failed with Kubernetes
+# 1.21.0-beta1.  Therefore the default version of KinD is "main"
+# for that, otherwise the latest stable release for which we then
+# list the officially supported images below.
+kind_version_default () {
+    case "${CSI_PROW_KUBERNETES_VERSION}" in
+        latest|master)
+            echo main;;
+        *)
+            echo v0.11.1;;
+    esac
+}
+
+# kind version to use. If the pre-installed version is different,
+# the desired version is downloaded from https://github.com/kubernetes-sigs/kind/releases
+# (if available), otherwise it is built from source.
+configvar CSI_PROW_KIND_VERSION "$(kind_version_default)" "kind"
+
+# kind images to use. Must match the kind version.
+# The release notes of each kind release list the supported images.
+configvar CSI_PROW_KIND_IMAGES "kindest/node:v1.22.0@sha256:b8bda84bb3a190e6e028b1760d277454a72267a5454b57db34437c34a588d047
+kindest/node:v1.21.1@sha256:69860bda5563ac81e3c0057d654b5253219618a22ec3a346306239bba8cfa1a6
+kindest/node:v1.20.7@sha256:cbeaf907fc78ac97ce7b625e4bf0de16e3ea725daf6b04f930bd14c67c671ff9
+kindest/node:v1.19.11@sha256:07db187ae84b4b7de440a73886f008cf903fcf5764ba8106a9fd5243d6f32729
+kindest/node:v1.18.19@sha256:7af1492e19b3192a79f606e43c35fb741e520d195f96399284515f077b3b622c
+kindest/node:v1.17.17@sha256:66f1d0d91a88b8a001811e2f1054af60eef3b669a9a74f9b6db871f2f1eeed00
+kindest/node:v1.16.15@sha256:83067ed51bf2a3395b24687094e283a7c7c865ccc12a8b1d7aa673ba0c5e8861
+kindest/node:v1.15.12@sha256:b920920e1eda689d9936dfcf7332701e80be12566999152626b2c9d730397a95
+kindest/node:v1.14.10@sha256:f8a66ef82822ab4f7569e91a5bccaf27bceee135c1457c512e54de8c6f7219f8" "kind images"
 
 # By default, this script tests sidecars with the CSI hostpath driver,
 # using the install_csi_driver function. That function depends on
@@ -171,8 +185,8 @@ configvar CSI_PROW_WORK "$(mkdir -p "$GOPATH/pkg" && mktemp -d "$GOPATH/pkg/csip
 #   CSI_PROW_DEPLOYMENT variable can be set in the
 #   .prow.sh of each component when there are breaking changes
 #   that require using a non-default deployment. The default
-#   is a deployment named "kubernetes-x.yy" (if available),
-#   otherwise "kubernetes-latest".
+#   is a deployment named "kubernetes-x.yy${CSI_PROW_DEPLOYMENT_SUFFIX}" (if available),
+#   otherwise "kubernetes-latest${CSI_PROW_DEPLOYMENT_SUFFIX}".
 #   "none" disables the deployment of the hostpath driver.
 #
 # When no deploy script is found (nothing in `deploy` directory,
@@ -184,6 +198,7 @@ configvar CSI_PROW_WORK "$(mkdir -p "$GOPATH/pkg" && mktemp -d "$GOPATH/pkg/csip
 configvar CSI_PROW_DRIVER_VERSION "v1.3.0" "CSI driver version"
 configvar CSI_PROW_DRIVER_REPO https://github.com/kubernetes-csi/csi-driver-host-path "CSI driver repo"
 configvar CSI_PROW_DEPLOYMENT "" "deployment"
+configvar CSI_PROW_DEPLOYMENT_SUFFIX "" "additional suffix in kubernetes-x.yy[suffix].yaml files"
 
 # The install_csi_driver function may work also for other CSI drivers,
 # as long as they follow the conventions of the CSI hostpath driver.
@@ -208,16 +223,7 @@ configvar CSI_PROW_DRIVER_CANARY_REGISTRY "gcr.io/k8s-staging-sig-storage" "regi
 # all generated files are present.
 #
 # CSI_PROW_E2E_REPO=none disables E2E testing.
-tag_from_version () {
-    version="$1"
-    shift
-    case "$version" in
-        latest) echo "master";;
-        release-*) echo "$version";;
-        *) echo "v$version";;
-    esac
-}
-configvar CSI_PROW_E2E_VERSION "$(tag_from_version "${CSI_PROW_KUBERNETES_VERSION}")"  "E2E version"
+configvar CSI_PROW_E2E_VERSION "$(version_to_git "${CSI_PROW_KUBERNETES_VERSION}")"  "E2E version"
 configvar CSI_PROW_E2E_REPO "https://github.com/kubernetes/kubernetes" "E2E repo"
 configvar CSI_PROW_E2E_IMPORT_PATH "k8s.io/kubernetes" "E2E package"
 
@@ -227,8 +233,8 @@ configvar CSI_PROW_E2E_IMPORT_PATH "k8s.io/kubernetes" "E2E package"
 # of the cluster. The alternative would have been to (cross-)compile csi-sanity
 # and install it inside the cluster, which is not necessarily easier.
 configvar CSI_PROW_SANITY_REPO https://github.com/kubernetes-csi/csi-test "csi-test repo"
-configvar CSI_PROW_SANITY_VERSION 5421d9f3c37be3b95b241b44a094a3db11bee789 "csi-test version" # latest master
-configvar CSI_PROW_SANITY_IMPORT_PATH github.com/kubernetes-csi/csi-test "csi-test package"
+configvar CSI_PROW_SANITY_VERSION v4.2.0 "csi-test version"
+configvar CSI_PROW_SANITY_PACKAGE_PATH github.com/kubernetes-csi/csi-test "csi-test package"
 configvar CSI_PROW_SANITY_SERVICE "hostpath-service" "Kubernetes TCP service name that exposes csi.sock"
 configvar CSI_PROW_SANITY_POD "csi-hostpathplugin-0" "Kubernetes pod with CSI driver"
 configvar CSI_PROW_SANITY_CONTAINER "hostpath" "Kubernetes container with CSI driver"
@@ -287,13 +293,26 @@ tests_need_alpha_cluster () {
     tests_enabled "parallel-alpha" "serial-alpha"
 }
 
+# Enabling mock tests adds the "CSI mock volume" tests from https://github.com/kubernetes/kubernetes/blob/HEAD/test/e2e/storage/csi_mock_volume.go
+# to the e2e.test invocations (serial, parallel, and the corresponding alpha variants).
+# When testing canary images, those get used instead of the images specified
+# in the e2e.test's normal YAML files.
+#
+# The default is to enable this for all jobs which use canary images
+# and the latest Kubernetes because those images will be used for mock
+# testing once they are released. Using them for mock testing with
+# older Kubernetes releases is too risky because the deployment files
+# can be very old (for example, still using a removed -provisioner
+# parameter in external-provisioner).
+configvar CSI_PROW_E2E_MOCK "$(if [ "${CSI_PROW_DRIVER_CANARY}" = "canary" ] && [ "${CSI_PROW_KUBERNETES_VERSION}" = "latest" ]; then echo true; else echo false; fi)" "enable CSI mock volume tests"
+
 # Regex for non-alpha, feature-tagged tests that should be run.
 #
 configvar CSI_PROW_E2E_FOCUS_LATEST '\[Feature:VolumeSnapshotDataSource\]' "non-alpha, feature-tagged tests for latest Kubernetes version"
 configvar CSI_PROW_E2E_FOCUS "$(get_versioned_variable CSI_PROW_E2E_FOCUS "${csi_prow_kubernetes_version_suffix}")" "non-alpha, feature-tagged tests"
 
 # Serial vs. parallel is always determined by these regular expressions.
-# Individual regular expressions are seperated by spaces for readability
+# Individual regular expressions are separated by spaces for readability
 # and expected to not contain spaces. Use dots instead. The complete
 # regex for Ginkgo will be created by joining the individual terms.
 configvar CSI_PROW_E2E_SERIAL '\[Serial\] \[Disruptive\]' "tags for serial E2E tests"
@@ -345,16 +364,25 @@ configvar CSI_SNAPSHOTTER_VERSION "$(default_csi_snapshotter_version)" "external
 # whether they can run with the current cluster provider, but until
 # they are, we filter them out by name. Like the other test selection
 # variables, this is again a space separated list of regular expressions.
-#
-# "different node" test skips can be removed once
-# https://github.com/kubernetes/kubernetes/pull/82678 has been backported
-# to all the K8s versions we test against
-configvar CSI_PROW_E2E_SKIP 'Disruptive|different\s+node' "tests that need to be skipped"
-
-# This is the directory for additional result files. Usually set by Prow, but
-# if not (for example, when invoking manually) it defaults to the work directory.
-configvar ARTIFACTS "${CSI_PROW_WORK}/artifacts" "artifacts"
-mkdir -p "${ARTIFACTS}"
+configvar CSI_PROW_E2E_SKIP 'Disruptive' "tests that need to be skipped"
+
+# This creates directories that are required for testing.
+ensure_paths () {
+    # Work directory. It has to allow running executables, therefore /tmp
+    # is avoided. Cleaning up after the script is intentionally left to
+    # the caller.
+    configvar CSI_PROW_WORK "$(mkdir -p "$GOPATH/pkg" && mktemp -d "$GOPATH/pkg/csiprow.XXXXXXXXXX")" "work directory"
+
+    # This is the directory for additional result files. Usually set by Prow, but
+    # if not (for example, when invoking manually) it defaults to the work directory.
+    configvar ARTIFACTS "${CSI_PROW_WORK}/artifacts" "artifacts"
+    mkdir -p "${ARTIFACTS}"
+
+    # For additional tools.
+    CSI_PROW_BIN="${CSI_PROW_WORK}/bin"
+    mkdir -p "${CSI_PROW_BIN}"
+    PATH="${CSI_PROW_BIN}:$PATH"
+}
 
 run () {
     echo "$(date) $(go version | sed -e 's/.*version \(go[^ ]*\).*/\1/') $(if [ "$(pwd)" != "${REPO_DIR}" ]; then pwd; fi)\$" "$@" >&2
@@ -374,11 +402,6 @@ die () {
     exit 1
 }
 
-# For additional tools.
-CSI_PROW_BIN="${CSI_PROW_WORK}/bin"
-mkdir -p "${CSI_PROW_BIN}"
-PATH="${CSI_PROW_BIN}:$PATH"
-
 # Ensure that PATH has the desired version of the Go tools, then run command given as argument.
 # Empty parameter uses the already installed Go. In Prow, that version is kept up-to-date by
 # bumping the container image regularly.
@@ -407,7 +430,7 @@ install_kind () {
         chmod u+x "${CSI_PROW_WORK}/bin/kind"
     else
         git_checkout https://github.com/kubernetes-sigs/kind "${GOPATH}/src/sigs.k8s.io/kind" "${CSI_PROW_KIND_VERSION}" --depth=1 &&
-        (cd "${GOPATH}/src/sigs.k8s.io/kind" && make install INSTALL_DIR="${CSI_PROW_WORK}/bin")
+        (cd "${GOPATH}/src/sigs.k8s.io/kind" && run_with_go "$CSI_PROW_GO_VERSION_KIND" make install INSTALL_DIR="${CSI_PROW_WORK}/bin")
     fi
 }
 
@@ -465,20 +488,22 @@ git_checkout () {
 
 # This clones a repo ("https://github.com/kubernetes/kubernetes")
 # in a certain location ("$GOPATH/src/k8s.io/kubernetes") at
-# a the head of a specific branch (i.e., release-1.13, master).
-# The directory cannot exist.
-git_clone_branch () {
-    local repo path branch parent
+# a the head of a specific branch (i.e., release-1.13, master),
+# tag (v1.20.0) or commit.
+#
+# The directory must not exist.
+git_clone () {
+    local repo path name parent
     repo="$1"
     shift
     path="$1"
     shift
-    branch="$1"
+    name="$1"
     shift
 
     parent="$(dirname "$path")"
     mkdir -p "$parent"
-    (cd "$parent" && run git clone --single-branch --branch "$branch" "$repo" "$path") || die "cloning $repo" failed
+    (cd "$parent" && run git clone --single-branch --branch "$name" "$repo" "$path") || die "cloning $repo" failed
     # This is useful for local testing or when switching between different revisions in the same
     # repo.
     (cd "$path" && run git clean -fdx) || die "failed to clean $path"
@@ -562,16 +587,12 @@ start_cluster () {
             if [ "$version" = "latest" ]; then
                 version=master
             fi
-            if ${CSI_PROW_USE_BAZEL}; then
-                type="bazel"
-            else
-                type="docker"
-            fi
-            git_clone_branch https://github.com/kubernetes/kubernetes "${CSI_PROW_WORK}/src/kubernetes" "$version" || die "checking out Kubernetes $version failed"
+            git_clone https://github.com/kubernetes/kubernetes "${CSI_PROW_WORK}/src/kubernetes" "$(version_to_git "$version")" || die "checking out Kubernetes $version failed"
 
             go_version="$(go_version_for_kubernetes "${CSI_PROW_WORK}/src/kubernetes" "$version")" || die "cannot proceed without knowing Go version for Kubernetes"
             # Changing into the Kubernetes source code directory is a workaround for https://github.com/kubernetes-sigs/kind/issues/1910
-            (cd "${CSI_PROW_WORK}/src/kubernetes" && run_with_go "$go_version" kind build node-image --image csiprow/node:latest --type="$type" --kube-root "${CSI_PROW_WORK}/src/kubernetes") || die "'kind build node-image' failed"
+            # shellcheck disable=SC2046
+            (cd "${CSI_PROW_WORK}/src/kubernetes" && run_with_go "$go_version" kind build node-image --image csiprow/node:latest --kube-root "${CSI_PROW_WORK}/src/kubernetes") || die "'kind build node-image' failed"
             csi_prow_kind_have_kubernetes=true
         fi
         image="csiprow/node:latest"
@@ -605,11 +626,16 @@ EOF
 
 # Deletes kind cluster inside a prow job
 delete_cluster_inside_prow_job() {
+    local name="$1"
+
     # Inside a real Prow job it is better to clean up at runtime
     # instead of leaving that to the Prow job cleanup code
     # because the later sometimes times out (https://github.com/kubernetes-csi/csi-release-tools/issues/24#issuecomment-554765872).
+    #
+    # This is also a good time to collect logs.
     if [ "$JOB_NAME" ]; then
         if kind get clusters | grep -q csi-prow; then
+            run kind export logs --name=csi-prow "${ARTIFACTS}/cluster-logs/$name"
             run kind delete cluster --name=csi-prow || die "kind delete failed"
         fi
         unset KUBECONFIG
@@ -619,24 +645,38 @@ delete_cluster_inside_prow_job() {
 # Looks for the deployment as specified by CSI_PROW_DEPLOYMENT and CSI_PROW_KUBERNETES_VERSION
 # in the given directory.
 find_deployment () {
-    local dir file
-    dir="$1"
+    local dir="$1"
+    local file
+
+    # major/minor without release- prefix.
+    local k8sver
+    # Ignore: See if you can use ${variable//search/replace} instead.
+    # shellcheck disable=SC2001
+    k8sver="$(echo "${CSI_PROW_KUBERNETES_VERSION}" | sed -e 's/^release-//' -e 's/\([0-9]*\)\.\([0-9]*\).*/\1.\2/')"
+
+    # Desired deployment, either specified completely, including version, or derived from other variables.
+    local deployment
+    deployment=${CSI_PROW_DEPLOYMENT:-kubernetes-${k8sver}${CSI_PROW_DEPLOYMENT_SUFFIX}}
 
-    # Fixed deployment name? Use it if it exists, otherwise fail.
+    # Fixed deployment name? Use it if it exists.
     if [ "${CSI_PROW_DEPLOYMENT}" ]; then
         file="$dir/${CSI_PROW_DEPLOYMENT}/deploy.sh"
-        if ! [ -e "$file" ]; then
-            return 1
+        if [ -e "$file" ]; then
+            echo "$file"
+            return 0
         fi
-        echo "$file"
-        return 0
+
+        # CSI_PROW_DEPLOYMENT=kubernetes-x.yy must be mapped to kubernetes-latest
+        # as fallback. Same for kubernetes-distributed-x.yy.
     fi
 
-    # Ignore: See if you can use ${variable//search/replace} instead.
-    # shellcheck disable=SC2001
-    file="$dir/kubernetes-$(echo "${CSI_PROW_KUBERNETES_VERSION}" | sed -e 's/\([0-9]*\)\.\([0-9]*\).*/\1.\2/')/deploy.sh"
+    file="$dir/${deployment}/deploy.sh"
     if ! [ -e "$file" ]; then
-        file="$dir/kubernetes-latest/deploy.sh"
+        # Replace the first xx.yy number with "latest", for example
+        # kubernetes-1.21-test -> kubernetes-latest-test.
+        # Ignore: See if you can use ${variable//search/replace} instead.
+        # shellcheck disable=SC2001
+        file="$dir/$(echo "$deployment" | sed -e 's/[0-9][0-9]*\.[0-9][0-9]*/latest/')/deploy.sh"
         if ! [ -e "$file" ]; then
             return 1
         fi
@@ -696,7 +736,7 @@ install_csi_driver () {
     fi
 }
 
-# Installs all nessesary snapshotter CRDs  
+# Installs all nessesary snapshotter CRDs
 install_snapshot_crds() {
   # Wait until volumesnapshot CRDs are in place.
   CRD_BASE_DIR="https://raw.githubusercontent.com/kubernetes-csi/external-snapshotter/${CSI_SNAPSHOTTER_VERSION}/client/config/crd"
@@ -743,7 +783,7 @@ install_snapshot_controller() {
     fi
     echo "$(date +%H:%M:%S)" "waiting for snapshot RBAC setup complete, attempt #$cnt"
 	cnt=$((cnt + 1))
-    sleep 10   
+    sleep 10
   done
 
   SNAPSHOT_CONTROLLER_YAML="${CONTROLLER_DIR}/deploy/kubernetes/snapshot-controller/setup-snapshot-controller.yaml"
@@ -756,7 +796,7 @@ install_snapshot_controller() {
       kind load docker-image --name csi-prow ${NEW_IMG} || die "could not load the snapshot-controller:csiprow image into the kind cluster"
 
       # deploy snapshot-controller
-      echo "Deploying snapshot-controller"
+      echo "Deploying snapshot-controller from ${SNAPSHOT_CONTROLLER_YAML} with $NEW_IMG."
       # Replace image in SNAPSHOT_CONTROLLER_YAML with snapshot-controller:csiprow and deploy
       # NOTE: This logic is similar to the logic here:
       # https://github.com/kubernetes-csi/csi-driver-host-path/blob/v1.4.0/deploy/util/deploy-hostpath.sh#L155
@@ -793,25 +833,37 @@ install_snapshot_controller() {
               echo "$modified"
               exit 1
           fi
-	  echo "kubectl apply -f ${SNAPSHOT_CONTROLLER_YAML}(modified)"
       done
+  elif [ "${CSI_PROW_DRIVER_CANARY}" = "canary" ]; then
+      echo "Deploying snapshot-controller from ${SNAPSHOT_CONTROLLER_YAML} with canary images."
+      yaml="$(kubectl apply --dry-run=client -o yaml -f "$SNAPSHOT_CONTROLLER_YAML")"
+      # Ignore: See if you can use ${variable//search/replace} instead.
+      # shellcheck disable=SC2001
+      modified="$(echo "$yaml" | sed -e "s;image: .*/\([^/:]*\):.*;image: ${CSI_PROW_DRIVER_CANARY_REGISTRY}/\1:canary;")"
+      diff <(echo "$yaml") <(echo "$modified")
+      if ! echo "$modified" | kubectl apply -f -; then
+          echo "modified version of $SNAPSHOT_CONTROLLER_YAML:"
+          echo "$modified"
+          exit 1
+      fi
   else
-      echo "kubectl apply -f ${CONTROLLER_DIR}/deploy/kubernetes/snapshot-controller/setup-snapshot-controller.yaml"
-      kubectl apply -f "${CONTROLLER_DIR}/deploy/kubernetes/snapshot-controller/setup-snapshot-controller.yaml"
+      echo "kubectl apply -f $SNAPSHOT_CONTROLLER_YAML"
+      kubectl apply -f "$SNAPSHOT_CONTROLLER_YAML"
   fi
 
   cnt=0
-  expected_running_pods=$(curl https://raw.githubusercontent.com/kubernetes-csi/external-snapshotter/"${CSI_SNAPSHOTTER_VERSION}"/deploy/kubernetes/snapshot-controller/setup-snapshot-controller.yaml | grep replicas | cut -d ':' -f 2-)
-  while [ "$(kubectl get pods -l app=snapshot-controller | grep 'Running' -c)" -lt "$expected_running_pods" ]; do
+  expected_running_pods=$(kubectl apply --dry-run=client -o "jsonpath={.spec.replicas}" -f "$SNAPSHOT_CONTROLLER_YAML")
+  expected_namespace=$(kubectl apply --dry-run=client -o "jsonpath={.metadata.namespace}" -f "$SNAPSHOT_CONTROLLER_YAML")
+  while [ "$(kubectl get pods -n "$expected_namespace" -l app=snapshot-controller | grep 'Running' -c)" -lt "$expected_running_pods" ]; do
     if [ $cnt -gt 30 ]; then
         echo "snapshot-controller pod status:"
-        kubectl describe pods -l app=snapshot-controller
+        kubectl describe pods -n "$expected_namespace" -l app=snapshot-controller
         echo >&2 "ERROR: snapshot controller not ready after over 5 min"
         exit 1
     fi
     echo "$(date +%H:%M:%S)" "waiting for snapshot controller deployment to complete, attempt #$cnt"
 	cnt=$((cnt + 1))
-    sleep 10   
+    sleep 10
   done
 }
 
@@ -856,6 +908,29 @@ start_loggers () {
     done
 }
 
+# Patches the image versions of test/e2e/testing-manifests/storage-csi/mock in the k/k
+# source code, if needed.
+patch_kubernetes () {
+    local source="$1" target="$2"
+
+    if [ "${CSI_PROW_DRIVER_CANARY}" = "canary" ]; then
+        # We cannot replace k8s.gcr.io/sig-storage with gcr.io/k8s-staging-sig-storage because
+        # e2e.test does not support it (see test/utils/image/manifest.go). Instead we
+        # invoke the e2e.test binary with KUBE_TEST_REPO_LIST set to a file that
+        # overrides that registry.
+        find "$source/test/e2e/testing-manifests/storage-csi/mock" -name '*.yaml' -print0 | xargs -0 sed -i -e 's;k8s.gcr.io/sig-storage/\(.*\):v.*;k8s.gcr.io/sig-storage/\1:canary;'
+        cat >"$target/e2e-repo-list" <<EOF
+sigStorageRegistry: gcr.io/k8s-staging-sig-storage
+EOF
+        cat >&2 <<EOF
+
+Using a modified version of k/k/test/e2e:
+$(cd "$source" && git diff 2>&1)
+
+EOF
+    fi
+}
+
 # Makes the E2E test suite binary available as "${CSI_PROW_WORK}/e2e.test".
 install_e2e () {
     if [ -e "${CSI_PROW_WORK}/e2e.test" ]; then
@@ -864,6 +939,7 @@ install_e2e () {
 
     git_checkout "${CSI_PROW_E2E_REPO}" "${GOPATH}/src/${CSI_PROW_E2E_IMPORT_PATH}" "${CSI_PROW_E2E_VERSION}" --depth=1 &&
     if [ "${CSI_PROW_E2E_IMPORT_PATH}" = "k8s.io/kubernetes" ]; then
+        patch_kubernetes "${GOPATH}/src/${CSI_PROW_E2E_IMPORT_PATH}" "${CSI_PROW_WORK}" &&
         go_version="${CSI_PROW_GO_VERSION_E2E:-$(go_version_for_kubernetes "${GOPATH}/src/${CSI_PROW_E2E_IMPORT_PATH}" "${CSI_PROW_E2E_VERSION}")}" &&
         run_with_go "$go_version" make WHAT=test/e2e/e2e.test "-C${GOPATH}/src/${CSI_PROW_E2E_IMPORT_PATH}" &&
         ln -s "${GOPATH}/src/${CSI_PROW_E2E_IMPORT_PATH}/_output/bin/e2e.test" "${CSI_PROW_WORK}"
@@ -879,8 +955,8 @@ install_sanity () (
         return
     fi
 
-    git_checkout "${CSI_PROW_SANITY_REPO}" "${GOPATH}/src/${CSI_PROW_SANITY_IMPORT_PATH}" "${CSI_PROW_SANITY_VERSION}" --depth=1 || die "checking out csi-sanity failed"
-    run_with_go "${CSI_PROW_GO_VERSION_SANITY}" go test -c -o "${CSI_PROW_WORK}/csi-sanity" "${CSI_PROW_SANITY_IMPORT_PATH}/cmd/csi-sanity" || die "building csi-sanity failed"
+    git_checkout "${CSI_PROW_SANITY_REPO}" "${GOPATH}/src/${CSI_PROW_SANITY_PACKAGE_PATH}" "${CSI_PROW_SANITY_VERSION}" --depth=1 || die "checking out csi-sanity failed"
+    ( cd "${GOPATH}/src/${CSI_PROW_SANITY_PACKAGE_PATH}/cmd/csi-sanity" && run_with_go "${CSI_PROW_GO_VERSION_SANITY}" go build -o "${CSI_PROW_WORK}/csi-sanity" ) || die "building csi-sanity failed"
 )
 
 # Captures pod output while running some other command.
@@ -909,25 +985,33 @@ run_e2e () (
     # the full Kubernetes E2E testsuite while only running a few tests.
     move_junit () {
         if ls "${ARTIFACTS}"/junit_[0-9]*.xml 2>/dev/null >/dev/null; then
-            run_filter_junit -t="External Storage" -o "${ARTIFACTS}/junit_${name}.xml" "${ARTIFACTS}"/junit_[0-9]*.xml && rm -f "${ARTIFACTS}"/junit_[0-9]*.xml
+            run_filter_junit -t="External.Storage|CSI.mock.volume" -o "${ARTIFACTS}/junit_${name}.xml" "${ARTIFACTS}"/junit_[0-9]*.xml && rm -f "${ARTIFACTS}"/junit_[0-9]*.xml
         fi
     }
     trap move_junit EXIT
 
     cd "${GOPATH}/src/${CSI_PROW_E2E_IMPORT_PATH}" &&
-    run_with_loggers ginkgo -v "$@" "${CSI_PROW_WORK}/e2e.test" -- -report-dir "${ARTIFACTS}" -storage.testdriver="${CSI_PROW_WORK}/test-driver.yaml"
+    run_with_loggers env KUBECONFIG="$KUBECONFIG" KUBE_TEST_REPO_LIST="$(if [ -e "${CSI_PROW_WORK}/e2e-repo-list" ]; then echo "${CSI_PROW_WORK}/e2e-repo-list"; fi)" ginkgo -v "$@" "${CSI_PROW_WORK}/e2e.test" -- -report-dir "${ARTIFACTS}" -storage.testdriver="${CSI_PROW_WORK}/test-driver.yaml"
 )
 
 # Run csi-sanity against installed CSI driver.
 run_sanity () (
     install_sanity || die "installing csi-sanity failed"
 
+    if [[ "${CSI_PROW_SANITY_POD}" =~ " " ]]; then
+        # Contains spaces, more complex than a simple pod name.
+        # Evaluate as a shell command.
+        pod=$(eval "${CSI_PROW_SANITY_POD}") || die "evaluation failed: CSI_PROW_SANITY_POD=${CSI_PROW_SANITY_POD}"
+    else
+        pod="${CSI_PROW_SANITY_POD}"
+    fi
+
     cat >"${CSI_PROW_WORK}/mkdir_in_pod.sh" <<EOF
 #!/bin/sh
-kubectl exec "${CSI_PROW_SANITY_POD}" -c "${CSI_PROW_SANITY_CONTAINER}" -- mkdir "\$@" && echo "\$@"
+kubectl exec "$pod" -c "${CSI_PROW_SANITY_CONTAINER}" -- mkdir "\$@" && echo "\$@"
 EOF
     # Using "rm -rf" as fallback for "rmdir" is a workaround for:
-    # Node Service 
+    # Node Service
     #     should work
     # /nvme/gopath.tmp/src/github.com/kubernetes-csi/csi-test/pkg/sanity/node.go:624
     # STEP: reusing connection to CSI driver at dns:///172.17.0.2:30896
@@ -949,11 +1033,29 @@ EOF
     # why it happened.
     cat >"${CSI_PROW_WORK}/rmdir_in_pod.sh" <<EOF
 #!/bin/sh
-if ! kubectl exec "${CSI_PROW_SANITY_POD}" -c "${CSI_PROW_SANITY_CONTAINER}" -- rmdir "\$@"; then
-    kubectl exec "${CSI_PROW_SANITY_POD}" -c "${CSI_PROW_SANITY_CONTAINER}" -- rm -rf "\$@"
+if ! kubectl exec "$pod" -c "${CSI_PROW_SANITY_CONTAINER}" -- rmdir "\$@"; then
+    kubectl exec "$pod" -c "${CSI_PROW_SANITY_CONTAINER}" -- rm -rf "\$@"
     exit 1
 fi
 EOF
+
+    cat >"${CSI_PROW_WORK}/checkdir_in_pod.sh" <<EOF
+#!/bin/sh
+CHECK_PATH=\$(cat <<SCRIPT
+if [ -f "\$@" ]; then
+    echo "file"
+elif [ -d "\$@" ]; then
+    echo "directory"
+elif [ -e "\$@" ]; then
+    echo "other"
+else
+    echo "not_found"
+fi
+SCRIPT
+)
+kubectl exec "$pod" -c "${CSI_PROW_SANITY_CONTAINER}" -- /bin/sh -c "\${CHECK_PATH}"
+EOF
+
     chmod u+x "${CSI_PROW_WORK}"/*dir_in_pod.sh
 
     # This cannot run in parallel, because -csi.junitfile output
@@ -969,6 +1071,7 @@ EOF
                      -csi.createmountpathcmd "${CSI_PROW_WORK}/mkdir_in_pod.sh" \
                      -csi.removestagingpathcmd "${CSI_PROW_WORK}/rmdir_in_pod.sh" \
                      -csi.removemountpathcmd "${CSI_PROW_WORK}/rmdir_in_pod.sh" \
+                     -csi.checkpathcmd "${CSI_PROW_WORK}/checkdir_in_pod.sh" \
 )
 
 ascii_to_xml () {
@@ -998,7 +1101,7 @@ make_test_to_junit () {
         echo "$line" # pass through
         if echo "$line" | grep -q "^### [^ ]*:$"; then
             if [ "$testname" ]; then
-                # previous test succesful
+                # previous test successful
                 echo "    </system-out>" >>"$out"
                 echo "  </testcase>" >>"$out"
             fi
@@ -1072,7 +1175,7 @@ make_test_to_junit () {
 # version_gt 1.3.1 v1.2.0  (returns true)
 # version_gt 1.1.1 release-1.2.0  (returns false)
 # version_gt 1.2.0 1.2.2  (returns false)
-function version_gt() { 
+function version_gt() {
     versions=$(for ver in "$@"; do ver=${ver#release-}; ver=${ver#kubernetes-}; echo "${ver#v}"; done)
     greaterVersion=${1#"release-"};
     greaterVersion=${greaterVersion#"kubernetes-"};
@@ -1084,6 +1187,9 @@ main () {
     local images ret
     ret=0
 
+    # Set up work directory.
+    ensure_paths
+
     images=
     if ${CSI_PROW_BUILD_JOB}; then
         # A successful build is required for testing.
@@ -1132,7 +1238,7 @@ main () {
                 if [ "$rbac_file_path" == "" ]; then
                     rbac_file_path="$(pwd)/deploy/kubernetes/rbac.yaml"
                 fi
-                
+
                 if [ -e "$rbac_file_path" ]; then
                     # This is one of those components which has its own RBAC rules (like external-provisioner).
                     # We are testing a locally built image and also want to test with the the current,
@@ -1143,6 +1249,12 @@ main () {
             done
         fi
 
+        # Run the external driver tests and optionally also mock tests.
+        local focus="External.Storage"
+        if "$CSI_PROW_E2E_MOCK"; then
+            focus="($focus|CSI.mock.volume)"
+        fi
+
         if tests_need_non_alpha_cluster; then
             start_cluster || die "starting the non-alpha cluster failed"
 
@@ -1150,6 +1262,7 @@ main () {
             install_snapshot_crds
             install_snapshot_controller
 
+
             # Installing the driver might be disabled.
             if ${CSI_PROW_DRIVER_INSTALL} "$images"; then
                 collect_cluster_info
@@ -1164,7 +1277,7 @@ main () {
                     # Ignore: Double quote to prevent globbing and word splitting.
                     # shellcheck disable=SC2086
                     if ! run_e2e parallel ${CSI_PROW_GINKO_PARALLEL} \
-                         -focus="External.Storage" \
+                         -focus="$focus" \
                          -skip="$(regex_join "${CSI_PROW_E2E_SERIAL}" "${CSI_PROW_E2E_ALPHA}" "${CSI_PROW_E2E_SKIP}")"; then
                         warn "E2E parallel failed"
                         ret=1
@@ -1174,7 +1287,7 @@ main () {
                     # Ignore: Double quote to prevent globbing and word splitting.
                     # shellcheck disable=SC2086
                     if ! run_e2e parallel-features ${CSI_PROW_GINKO_PARALLEL} \
-                         -focus="External.Storage.*($(regex_join "${CSI_PROW_E2E_FOCUS}"))" \
+                         -focus="$focus.*($(regex_join "${CSI_PROW_E2E_FOCUS}"))" \
                          -skip="$(regex_join "${CSI_PROW_E2E_SERIAL}")"; then
                         warn "E2E parallel features failed"
                         ret=1
@@ -1183,14 +1296,14 @@ main () {
 
                 if tests_enabled "serial"; then
                     if ! run_e2e serial \
-                         -focus="External.Storage.*($(regex_join "${CSI_PROW_E2E_SERIAL}"))" \
+                         -focus="$focus.*($(regex_join "${CSI_PROW_E2E_SERIAL}"))" \
                          -skip="$(regex_join "${CSI_PROW_E2E_ALPHA}" "${CSI_PROW_E2E_SKIP}")"; then
                         warn "E2E serial failed"
                         ret=1
                     fi
                 fi
             fi
-            delete_cluster_inside_prow_job
+            delete_cluster_inside_prow_job non-alpha
         fi
 
         if tests_need_alpha_cluster && [ "${CSI_PROW_E2E_ALPHA_GATES}" ]; then
@@ -1209,7 +1322,7 @@ main () {
                     # Ignore: Double quote to prevent globbing and word splitting.
                     # shellcheck disable=SC2086
                     if ! run_e2e parallel-alpha ${CSI_PROW_GINKO_PARALLEL} \
-                         -focus="External.Storage.*($(regex_join "${CSI_PROW_E2E_ALPHA}"))" \
+                         -focus="$focus.*($(regex_join "${CSI_PROW_E2E_ALPHA}"))" \
                          -skip="$(regex_join "${CSI_PROW_E2E_SERIAL}" "${CSI_PROW_E2E_SKIP}")"; then
                         warn "E2E parallel alpha failed"
                         ret=1
@@ -1218,14 +1331,14 @@ main () {
 
                 if tests_enabled "serial-alpha"; then
                     if ! run_e2e serial-alpha \
-                         -focus="External.Storage.*(($(regex_join "${CSI_PROW_E2E_SERIAL}")).*($(regex_join "${CSI_PROW_E2E_ALPHA}"))|($(regex_join "${CSI_PROW_E2E_ALPHA}")).*($(regex_join "${CSI_PROW_E2E_SERIAL}")))" \
+                         -focus="$focus.*(($(regex_join "${CSI_PROW_E2E_SERIAL}")).*($(regex_join "${CSI_PROW_E2E_ALPHA}"))|($(regex_join "${CSI_PROW_E2E_ALPHA}")).*($(regex_join "${CSI_PROW_E2E_SERIAL}")))" \
                          -skip="$(regex_join "${CSI_PROW_E2E_SKIP}")"; then
                         warn "E2E serial alpha failed"
                         ret=1
                     fi
                 fi
             fi
-            delete_cluster_inside_prow_job
+            delete_cluster_inside_prow_job alpha
         fi
     fi
 
@@ -1245,6 +1358,9 @@ gcr_cloud_build () {
     # Required for "docker buildx build --push".
     gcloud auth configure-docker
 
+    # Might not be needed here, but call it just in case.
+    ensure_paths
+
     if find . -name Dockerfile | grep -v ^./vendor | xargs --no-run-if-empty cat | grep -q ^RUN; then
         # Needed for "RUN" steps on non-linux/amd64 platforms.
         # See https://github.com/multiarch/qemu-user-static#getting-started

pull-test.sh

+#! /bin/sh
+
+# Copyright 2021 The Kubernetes Authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# This script is called by pull Prow jobs for the csi-release-tools
+# repo to ensure that the changes in the PR work when imported into
+# some other repo.
+
+set -ex
+
+# It must be called inside the updated csi-release-tools repo.
+CSI_RELEASE_TOOLS_DIR="$(pwd)"
+
+# Update the other repo.
+cd "$PULL_TEST_REPO_DIR"
+git subtree pull --squash --prefix=release-tools "$CSI_RELEASE_TOOLS_DIR" master
+git log -n2
+
+# Now fall through to testing.
+exec ./.prow.sh

travis.yml

-language: go
-sudo: required
-services:
-  - docker
-git:
-  depth: false
-matrix:
-  include:
-  - go: 1.15
-before_script:
-- mkdir -p bin
-- wget https://github.com/golang/dep/releases/download/v0.5.1/dep-linux-amd64 -O bin/dep
-- chmod u+x bin/dep
-- export PATH=$PWD/bin:$PATH
-script:
-- make -k all test GOFLAGS_VENDOR=$( [ -d vendor ] && echo '-mod=vendor' )
-after_success:
-  - if [ "${TRAVIS_PULL_REQUEST}" == "false" ]; then
-      docker login -u "${DOCKER_USERNAME}" -p "${DOCKER_PASSWORD}" quay.io;
-      make push GOFLAGS_VENDOR=$( [ -d vendor ] && echo '-mod=vendor' );
-    fi

verify-boilerplate.sh

+#!/bin/bash
+
+# Copyright 2014 The Kubernetes Authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+set -o errexit
+set -o nounset
+set -o pipefail
+
+echo "Verifying boilerplate"
+
+if [[ -z "$(command -v python)" ]]; then
+  echo "Cannot find python. Make link to python3..."
+  update-alternatives --install /usr/bin/python python /usr/bin/python3 1
+fi
+
+# The csi-release-tools directory (absolute path).
+TOOLS="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd -P)"
+
+# Directory to check. Default is the parent of the tools themselves.
+ROOT="${1:-${TOOLS}/..}"
+
+boiler="${TOOLS}/boilerplate/boilerplate.py"
+
+mapfile -t files_need_boilerplate < <("${boiler}" --rootdir="${ROOT}" --verbose)
+
+# Run boilerplate.py unit tests
+unitTestOut="$(mktemp)"
+trap cleanup EXIT
+cleanup() {
+	rm "${unitTestOut}"
+}
+
+# Run boilerplate check
+if [[ ${#files_need_boilerplate[@]} -gt 0 ]]; then
+  for file in "${files_need_boilerplate[@]}"; do
+    echo "Boilerplate header is wrong for: ${file}"
+  done
+
+  exit 1
+fi
+
+echo "Done"

verify-go-version.sh

@@ -29,8 +29,9 @@ die () {
 version=$("$GO" version) || die "determining version of $GO failed"
 # shellcheck disable=SC2001
 majorminor=$(echo "$version" | sed -e 's/.*go\([0-9]*\)\.\([0-9]*\).*/\1.\2/')
-# shellcheck disable=SC2001
-expected=$(grep "^ *- go:" "release-tools/travis.yml" | sed -e 's/.*go: *\([0-9]*\)\.\([0-9]*\).*/\1.\2/')
+# SC1091: Not following: release-tools/prow.sh was not specified as input (see shellcheck -x).
+# shellcheck disable=SC1091
+expected=$(. release-tools/prow.sh >/dev/null && echo "$CSI_PROW_GO_VERSION_BUILD")
 
 if [ "$majorminor" != "$expected" ]; then
     cat >&2 <<EOF

verify-shellcheck.sh

@@ -84,7 +84,7 @@ done < <(find . -name "*.sh" \
 # detect if the host machine has the required shellcheck version installed
 # if so, we will use that instead.
 HAVE_SHELLCHECK=false
-if which shellcheck &>/dev/null; then
+if command -v shellcheck &>/dev/null; then
   detected_version="$(shellcheck --version | grep 'version: .*')"
   if [[ "${detected_version}" = "version: ${SHELLCHECK_VERSION}" ]]; then
     HAVE_SHELLCHECK=true

verify-spelling.sh

+#!/usr/bin/env bash
+
+# Copyright 2019 The Kubernetes Authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+set -o errexit
+set -o nounset
+set -o pipefail
+
+TOOL_VERSION="v0.3.4"
+
+# The csi-release-tools directory (absolute path).
+TOOLS="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd -P)"
+
+# Directory to check. Default is the parent of the tools themselves.
+ROOT="${1:-${TOOLS}/..}"
+
+# create a temporary directory
+TMP_DIR=$(mktemp -d)
+
+# cleanup
+exitHandler() (
+  echo "Cleaning up..."
+  rm -rf "${TMP_DIR}"
+)
+trap exitHandler EXIT
+
+if [[ -z "$(command -v misspell)" ]]; then
+  echo "Cannot find misspell. Installing misspell..."
+  # perform go get in a temp dir as we are not tracking this version in a go module
+  # if we do the go get in the repo, it will create / update a go.mod and go.sum
+  cd "${TMP_DIR}"
+  GO111MODULE=on GOBIN="${TMP_DIR}" go get "github.com/client9/misspell/cmd/misspell@${TOOL_VERSION}"
+  export PATH="${TMP_DIR}:${PATH}"
+fi
+
+# check spelling
+RES=0
+echo "Checking spelling..."
+ERROR_LOG="${TMP_DIR}/errors.log"
+cd "${ROOT}"
+git ls-files | grep -v vendor | xargs misspell > "${ERROR_LOG}"
+if [[ -s "${ERROR_LOG}" ]]; then
+  sed 's/^/error: /' "${ERROR_LOG}" # add 'error' to each line to highlight in e2e status
+  echo "Found spelling errors!"
+  RES=1
+fi
+exit "${RES}"

verify-subtree.sh

 #! /bin/sh -e
-#
+
 # Copyright 2019 The Kubernetes Authors.
 #
 # Licensed under the Apache License, Version 2.0 (the "License");