From 87ca82891ac8e03b42702b2a853011b7e505b6ab Mon Sep 17 00:00:00 2001 From: Bhukya Date: Mon, 19 Oct 2026 21:42:37 +0000 Subject: [PATCH 01/10] Add databricks pipelines test command Submits a test-only pipeline update with optional test selections and pytest arguments. Relative selections resolve to deployed workspace paths through the bundle sync root, or against the pipeline's root_path when the pipeline is identified with --pipeline-id. Only -k and -m are accepted in --pytest-arg. Co-authored-by: Isaac --- .nextchanges/cli/pipelines-test.md | 1 + .../pipelines/databricks-cli-help/output.txt | 1 + .../pipelines/test/arguments/out.test.toml | 1 + .../pipelines/test/arguments/output.txt | 61 ++++ acceptance/pipelines/test/arguments/script | 14 + .../pipelines/test/pipeline-id/out.test.toml | 1 + .../pipelines/test/pipeline-id/output.txt | 46 +++ acceptance/pipelines/test/pipeline-id/script | 15 + .../pipelines/test/selection/databricks.yml | 10 + .../pipelines/test/selection/out.test.toml | 1 + .../pipelines/test/selection/output.txt | 94 ++++++ .../pipelines/test/selection/pipeline_file.py | 6 + acceptance/pipelines/test/selection/script | 22 ++ .../test/selection/tests/test_orders.py | 6 + acceptance/pipelines/test/test.toml | 3 + bundle/root.go | 7 + bundle/root_test.go | 21 ++ cmd/pipelines/commands.go | 1 + cmd/pipelines/test.go | 317 ++++++++++++++++++ cmd/pipelines/test_test.go | 116 +++++++ 20 files changed, 744 insertions(+) create mode 100644 .nextchanges/cli/pipelines-test.md create mode 100644 acceptance/pipelines/test/arguments/out.test.toml create mode 100644 acceptance/pipelines/test/arguments/output.txt create mode 100644 acceptance/pipelines/test/arguments/script create mode 100644 acceptance/pipelines/test/pipeline-id/out.test.toml create mode 100644 acceptance/pipelines/test/pipeline-id/output.txt create mode 100644 acceptance/pipelines/test/pipeline-id/script create mode 100644 acceptance/pipelines/test/selection/databricks.yml create mode 100644 acceptance/pipelines/test/selection/out.test.toml create mode 100644 acceptance/pipelines/test/selection/output.txt create mode 100644 acceptance/pipelines/test/selection/pipeline_file.py create mode 100644 acceptance/pipelines/test/selection/script create mode 100644 acceptance/pipelines/test/selection/tests/test_orders.py create mode 100644 acceptance/pipelines/test/test.toml create mode 100644 cmd/pipelines/test.go create mode 100644 cmd/pipelines/test_test.go diff --git a/.nextchanges/cli/pipelines-test.md b/.nextchanges/cli/pipelines-test.md new file mode 100644 index 00000000000..f7ac2beee9b --- /dev/null +++ b/.nextchanges/cli/pipelines-test.md @@ -0,0 +1 @@ +* Add the `databricks pipelines test` command, which runs a pipeline's tests in a test-only update. diff --git a/acceptance/pipelines/databricks-cli-help/output.txt b/acceptance/pipelines/databricks-cli-help/output.txt index 8514fa33b2b..3fb01646d00 100644 --- a/acceptance/pipelines/databricks-cli-help/output.txt +++ b/acceptance/pipelines/databricks-cli-help/output.txt @@ -30,6 +30,7 @@ Main Commands run Run a pipeline show Preview a table's columns and sample rows stop Stop a pipeline + test Run the tests of a pipeline Management Commands apply-environment *Public Preview* Apply the latest environment to the pipeline. diff --git a/acceptance/pipelines/test/arguments/out.test.toml b/acceptance/pipelines/test/arguments/out.test.toml new file mode 100644 index 00000000000..18b1a88417e --- /dev/null +++ b/acceptance/pipelines/test/arguments/out.test.toml @@ -0,0 +1 @@ +Cloud = false diff --git a/acceptance/pipelines/test/arguments/output.txt b/acceptance/pipelines/test/arguments/output.txt new file mode 100644 index 00000000000..dc32fe74d13 --- /dev/null +++ b/acceptance/pipelines/test/arguments/output.txt @@ -0,0 +1,61 @@ + +>>> [CLI] pipelines test --help +Run the tests of a pipeline in a test-only update. + +PIPELINE_KEY is the unique name of the pipeline, as defined in its YAML file. +It is required when test selections are given. If there is only one pipeline +in the project and no arguments are given, it is selected automatically. +Outside a project, identify the pipeline with --pipeline-id; all positional +arguments are then test selections. + +TEST_SELECTION is a test directory, file, or pytest test ID, for example +tests/test_orders.py::test_backfill. Relative paths are converted to their +deployed workspace paths, or outside a project resolved against the pipeline's +root_path. Paths that start with / are workspace paths. Without a selection, +all of the pipeline's tests run. + +Each --pytest-arg passes one argument to pytest, in the order given. Only -k +and -m are supported, for example: + + databricks pipelines test my_pipeline --pytest-arg=-k --pytest-arg='orders and not slow' + +Usage: + databricks pipelines test [PIPELINE_KEY] [TEST_SELECTION...] [flags] [--pytest-arg=ARG ...] + +Pipeline Test Flags: + --pipeline-id string ID of the pipeline to test, instead of a PIPELINE_KEY. + --pytest-arg stringArray Argument to pass to pytest (repeatable). Only -k and -m are supported. + +Flags: + -h, --help help for test + +Global Flags: + --debug enable debug logging + -o, --output type output type: text or json (default text) + -p, --profile string ~/.databrickscfg profile + -t, --target string bundle target to use (if applicable) + --var strings set values for variables defined in project config. Example: --var="foo=bar" + +=== pytest arguments must be passed with --pytest-arg +>>> musterr [CLI] pipelines test --pipeline-id 1234 -k orders +Error: -k is a pytest argument: pass pytest arguments with --pytest-arg, for example --pytest-arg=-k --pytest-arg='orders and not slow' + +>>> musterr [CLI] pipelines test --pipeline-id 1234 -msmoke +Error: -m is a pytest argument: pass pytest arguments with --pytest-arg, for example --pytest-arg=-k --pytest-arg='orders and not slow' + +>>> musterr [CLI] pipelines test --pipeline-id 1234 -- -k orders +Error: the -- separator is not supported: pass pytest arguments with --pytest-arg, for example --pytest-arg=-k --pytest-arg='orders and not slow' + +=== Only -k and -m are supported +>>> musterr [CLI] pipelines test --pipeline-id 1234 --pytest-arg=-x +Error: unsupported pytest argument "-x": only -k and -m are supported + +>>> musterr [CLI] pipelines test --pipeline-id 1234 --pytest-arg=-k +Error: pytest argument -k requires a value: pass it in the next --pytest-arg + +>>> musterr [CLI] pipelines test --pipeline-id 1234 --pytest-arg=tests/test_orders.py +Error: unsupported pytest argument "tests/test_orders.py": pass test paths and test IDs as TEST_SELECTION arguments + +=== JSON output is not supported +>>> musterr [CLI] pipelines test --pipeline-id 1234 -o json +Error: --output json is not supported by pipelines test diff --git a/acceptance/pipelines/test/arguments/script b/acceptance/pipelines/test/arguments/script new file mode 100644 index 00000000000..dfe93d5c4b6 --- /dev/null +++ b/acceptance/pipelines/test/arguments/script @@ -0,0 +1,14 @@ +trace $CLI pipelines test --help + +title "pytest arguments must be passed with --pytest-arg" +trace musterr $CLI pipelines test --pipeline-id 1234 -k orders +trace musterr $CLI pipelines test --pipeline-id 1234 -msmoke +trace musterr $CLI pipelines test --pipeline-id 1234 -- -k orders + +title "Only -k and -m are supported" +trace musterr $CLI pipelines test --pipeline-id 1234 --pytest-arg=-x +trace musterr $CLI pipelines test --pipeline-id 1234 --pytest-arg=-k +trace musterr $CLI pipelines test --pipeline-id 1234 --pytest-arg=tests/test_orders.py + +title "JSON output is not supported" +trace musterr $CLI pipelines test --pipeline-id 1234 -o json diff --git a/acceptance/pipelines/test/pipeline-id/out.test.toml b/acceptance/pipelines/test/pipeline-id/out.test.toml new file mode 100644 index 00000000000..18b1a88417e --- /dev/null +++ b/acceptance/pipelines/test/pipeline-id/out.test.toml @@ -0,0 +1 @@ +Cloud = false diff --git a/acceptance/pipelines/test/pipeline-id/output.txt b/acceptance/pipelines/test/pipeline-id/output.txt new file mode 100644 index 00000000000..9138ee2e806 --- /dev/null +++ b/acceptance/pipelines/test/pipeline-id/output.txt @@ -0,0 +1,46 @@ + +=== Relative test selections resolve against the pipeline root_path +>>> [CLI] pipelines test --pipeline-id [UUID] tests/test_orders.py::test_backfill /Workspace/Shared/tests +Pipeline: [UUID] +Update: [UUID] + +>>> print_requests.py //updates +{ + "method": "POST", + "path": "/api/2.0/pipelines/[UUID]/updates", + "body": { + "test_only": true, + "test_options": { + "test_selection": [ + "/Workspace/Users/someone@example.com/orders/tests/test_orders.py::test_backfill", + "/Workspace/Shared/tests" + ] + } + } +} + +=== Without a root_path, test selections must be workspace paths +>>> musterr [CLI] pipelines test --pipeline-id [UUID] tests +Error: cannot resolve test selection "tests": the pipeline has no root_path; use an absolute workspace path such as /Workspace/Users/someone@example.com/tests + +>>> [CLI] pipelines test --pipeline-id [UUID] /Workspace/Shared/tests/test_shared.py::test_x +Pipeline: [UUID] +Update: [UUID] + +>>> print_requests.py //updates +{ + "method": "POST", + "path": "/api/2.0/pipelines/[UUID]/updates", + "body": { + "test_only": true, + "test_options": { + "test_selection": [ + "/Workspace/Shared/tests/test_shared.py::test_x" + ] + } + } +} + +=== Outside a project, --pipeline-id is required +>>> musterr [CLI] pipelines test tests +Error: outside a project, identify the pipeline with --pipeline-id diff --git a/acceptance/pipelines/test/pipeline-id/script b/acceptance/pipelines/test/pipeline-id/script new file mode 100644 index 00000000000..2ef512b1210 --- /dev/null +++ b/acceptance/pipelines/test/pipeline-id/script @@ -0,0 +1,15 @@ +pipeline_id=$($CLI pipelines create --json '{"name": "orders", "root_path": "/Workspace/Users/someone@example.com/orders"}' | jq -r .pipeline_id) +no_root_pipeline_id=$($CLI pipelines create --json '{"name": "no-root"}' | jq -r .pipeline_id) +rm "$OUT_REQUESTS" + +title "Relative test selections resolve against the pipeline root_path" +trace $CLI pipelines test --pipeline-id $pipeline_id tests/test_orders.py::test_backfill /Workspace/Shared/tests +trace print_requests.py //updates + +title "Without a root_path, test selections must be workspace paths" +trace musterr $CLI pipelines test --pipeline-id $no_root_pipeline_id tests +trace $CLI pipelines test --pipeline-id $no_root_pipeline_id /Workspace/Shared/tests/test_shared.py::test_x +trace print_requests.py //updates + +title "Outside a project, --pipeline-id is required" +trace musterr $CLI pipelines test tests diff --git a/acceptance/pipelines/test/selection/databricks.yml b/acceptance/pipelines/test/selection/databricks.yml new file mode 100644 index 00000000000..0cf849a7919 --- /dev/null +++ b/acceptance/pipelines/test/selection/databricks.yml @@ -0,0 +1,10 @@ +bundle: + name: test-pipeline-test-selection + +resources: + pipelines: + my_pipeline: + name: test-pipeline-test-selection + libraries: + - file: + path: pipeline_file.py diff --git a/acceptance/pipelines/test/selection/out.test.toml b/acceptance/pipelines/test/selection/out.test.toml new file mode 100644 index 00000000000..18b1a88417e --- /dev/null +++ b/acceptance/pipelines/test/selection/out.test.toml @@ -0,0 +1 @@ +Cloud = false diff --git a/acceptance/pipelines/test/selection/output.txt b/acceptance/pipelines/test/selection/output.txt new file mode 100644 index 00000000000..8b3bca77bc9 --- /dev/null +++ b/acceptance/pipelines/test/selection/output.txt @@ -0,0 +1,94 @@ + +>>> [CLI] pipelines deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-pipeline-test-selection/default/files... +Created pipelines.my_pipeline +Files: 3 uploaded, 0 deleted +Resources: 1 created, 0 changed, 0 deleted, 0 unchanged +View your pipeline my_pipeline here: [DATABRICKS_URL]/pipelines/[UUID]?w=[WORKSPACE_ID] + +=== Run all tests of the only pipeline +>>> [CLI] pipelines test +Recommendation: This command runs the last deployed version of the code + +If you've made local changes, run 'databricks pipelines deploy' first to ensure they are included. + +Pipeline: my_pipeline ([UUID]) +Update: [UUID] + +>>> print_requests.py //updates +{ + "method": "POST", + "path": "/api/2.0/pipelines/[UUID]/updates", + "body": { + "test_only": true + } +} + +=== Select tests by relative path and test ID, with pytest arguments +>>> [CLI] pipelines test my_pipeline tests tests/test_orders.py::test_backfill /Workspace/Shared/tests/test_shared.py --pytest-arg=-k --pytest-arg=orders and not slow --pytest-arg=-msmoke,fast +Recommendation: This command runs the last deployed version of the code + +If you've made local changes, run 'databricks pipelines deploy' first to ensure they are included. + +Pipeline: my_pipeline ([UUID]) +Update: [UUID] + +>>> print_requests.py //updates +{ + "method": "POST", + "path": "/api/2.0/pipelines/[UUID]/updates", + "body": { + "test_only": true, + "test_options": { + "test_selection": [ + "/Workspace/Users/[USERNAME]/.bundle/test-pipeline-test-selection/default/files/tests", + "/Workspace/Users/[USERNAME]/.bundle/test-pipeline-test-selection/default/files/tests/test_orders.py::test_backfill", + "/Workspace/Shared/tests/test_shared.py" + ], + "pytest_args": [ + "-k", + "orders and not slow", + "-msmoke,fast" + ] + } + } +} + +=== Relative paths resolve from the working directory +>>> [CLI] pipelines test my_pipeline test_orders.py::test_valid_order +Recommendation: This command runs the last deployed version of the code + +If you've made local changes, run 'databricks pipelines deploy' first to ensure they are included. + +Pipeline: my_pipeline ([UUID]) +Update: [UUID] + +>>> print_requests.py //updates +{ + "method": "POST", + "path": "/api/2.0/pipelines/[UUID]/updates", + "body": { + "test_only": true, + "test_options": { + "test_selection": [ + "/Workspace/Users/[USERNAME]/.bundle/test-pipeline-test-selection/default/files/tests/test_orders.py::test_valid_order" + ] + } + } +} + +=== Test selections require the pipeline KEY first +>>> musterr [CLI] pipelines test tests/test_orders.py +Recommendation: This command runs the last deployed version of the code + +If you've made local changes, run 'databricks pipelines deploy' first to ensure they are included. + +Error: resource with key "tests/test_orders.py" not found; the first argument must be a pipeline KEY when test selections are given + +=== Test selections must be in the synchronized files +>>> musterr [CLI] pipelines test my_pipeline ../test_outside.py +Recommendation: This command runs the last deployed version of the code + +If you've made local changes, run 'databricks pipelines deploy' first to ensure they are included. + +Error: cannot resolve test selection "../test_outside.py": [TEST_TMP_DIR]_PARENT/test_outside.py is outside the synchronized bundle files in [TEST_TMP_DIR] diff --git a/acceptance/pipelines/test/selection/pipeline_file.py b/acceptance/pipelines/test/selection/pipeline_file.py new file mode 100644 index 00000000000..4a7a3a1878e --- /dev/null +++ b/acceptance/pipelines/test/selection/pipeline_file.py @@ -0,0 +1,6 @@ +from pyspark import pipelines as dp + + +@dp.table +def orders(): + return spark.range(10) diff --git a/acceptance/pipelines/test/selection/script b/acceptance/pipelines/test/selection/script new file mode 100644 index 00000000000..22348d2962f --- /dev/null +++ b/acceptance/pipelines/test/selection/script @@ -0,0 +1,22 @@ +trace $CLI pipelines deploy +rm "$OUT_REQUESTS" + +title "Run all tests of the only pipeline" +trace $CLI pipelines test +trace print_requests.py //updates + +title "Select tests by relative path and test ID, with pytest arguments" +trace $CLI pipelines test my_pipeline tests tests/test_orders.py::test_backfill /Workspace/Shared/tests/test_shared.py --pytest-arg=-k --pytest-arg='orders and not slow' --pytest-arg=-msmoke,fast +trace print_requests.py //updates + +title "Relative paths resolve from the working directory" +cd tests +trace $CLI pipelines test my_pipeline test_orders.py::test_valid_order +cd .. +trace print_requests.py //updates + +title "Test selections require the pipeline KEY first" +trace musterr $CLI pipelines test tests/test_orders.py + +title "Test selections must be in the synchronized files" +trace musterr $CLI pipelines test my_pipeline ../test_outside.py diff --git a/acceptance/pipelines/test/selection/tests/test_orders.py b/acceptance/pipelines/test/selection/tests/test_orders.py new file mode 100644 index 00000000000..6058cdd0bba --- /dev/null +++ b/acceptance/pipelines/test/selection/tests/test_orders.py @@ -0,0 +1,6 @@ +def test_valid_order(): + assert True + + +def test_backfill(): + assert True diff --git a/acceptance/pipelines/test/test.toml b/acceptance/pipelines/test/test.toml new file mode 100644 index 00000000000..9fc0358f8ac --- /dev/null +++ b/acceptance/pipelines/test/test.toml @@ -0,0 +1,3 @@ +RecordRequests = true +Ignore = ['.databricks'] +Env.MSYS_NO_PATHCONV = "1" diff --git a/bundle/root.go b/bundle/root.go index 5f7317a5e99..c41c52edfb7 100644 --- a/bundle/root.go +++ b/bundle/root.go @@ -70,6 +70,13 @@ func mustGetRoot(ctx context.Context) (string, diag.Diagnostics) { return getRootWithTraversal() } +// HasRoot reports whether a bundle root is configured or can be found without loading it. +// An invalid bundle root environment variable counts as a root so that loading reports the error. +func HasRoot(ctx context.Context) bool { + path, diags := tryGetRoot(ctx) + return path != "" || diags.HasError() +} + // tryGetRoot returns a bundle root or an empty string if one cannot be found. func tryGetRoot(ctx context.Context) (string, diag.Diagnostics) { // Note: an invalid value in the environment variable is still an error. diff --git a/bundle/root_test.go b/bundle/root_test.go index 27109f2aeff..20b160b8b40 100644 --- a/bundle/root_test.go +++ b/bundle/root_test.go @@ -110,3 +110,24 @@ func TestRootLookupError(t *testing.T) { require.True(t, diags.HasError()) require.Contains(t, diags[0].Summary, "Unable to locate the bundle root") } + +func TestHasRoot(t *testing.T) { + ctx := t.Context() + + // Have to set then unset to allow the testing package to revert it to its original value. + t.Setenv(env.RootVariable, "") + os.Unsetenv(env.RootVariable) + + dir := t.TempDir() + t.Chdir(dir) + require.False(t, HasRoot(ctx)) + + f, err := os.Create(filepath.Join(dir, config.FileNames[0])) + require.NoError(t, err) + f.Close() + require.True(t, HasRoot(ctx)) + + // An invalid environment variable is reported when the bundle is loaded. + t.Setenv(env.RootVariable, filepath.Join(dir, "doesntexist")) + require.True(t, HasRoot(ctx)) +} diff --git a/cmd/pipelines/commands.go b/cmd/pipelines/commands.go index 6a6b6931313..24882f97864 100644 --- a/cmd/pipelines/commands.go +++ b/cmd/pipelines/commands.go @@ -17,6 +17,7 @@ func Commands() []*cobra.Command { destroyCommand(), runCommand(), dryRunCommand(), + testCommand(), historyCommand(), logsCommand(), openCommand(), diff --git a/cmd/pipelines/test.go b/cmd/pipelines/test.go new file mode 100644 index 00000000000..7a9da8bef9d --- /dev/null +++ b/cmd/pipelines/test.go @@ -0,0 +1,317 @@ +package pipelines + +import ( + "context" + "errors" + "fmt" + "io" + "net/http" + "os" + "path" + "path/filepath" + "strings" + + "github.com/databricks/cli/bundle" + "github.com/databricks/cli/bundle/resources" + "github.com/databricks/cli/cmd/bundle/utils" + "github.com/databricks/cli/cmd/root" + "github.com/databricks/cli/libs/auth" + "github.com/databricks/cli/libs/cmdctx" + "github.com/databricks/cli/libs/cmdgroup" + "github.com/databricks/cli/libs/flags" + "github.com/databricks/cli/libs/logdiag" + databricks "github.com/databricks/databricks-sdk-go" + "github.com/databricks/databricks-sdk-go/client" + "github.com/databricks/databricks-sdk-go/service/pipelines" + "github.com/spf13/cobra" + "github.com/spf13/pflag" +) + +// startTestUpdateRequest is the Start Update body of a test run. +// test_only and test_options are not in the Go SDK yet, so the request is sent directly. +type startTestUpdateRequest struct { + TestOnly bool `json:"test_only"` + TestOptions *testOptions `json:"test_options,omitempty"` +} + +type testOptions struct { + TestSelection []string `json:"test_selection,omitempty"` + PytestArgs []string `json:"pytest_args,omitempty"` +} + +func newStartTestUpdateRequest(selection, pytestArgs []string) startTestUpdateRequest { + req := startTestUpdateRequest{TestOnly: true} + if len(selection) > 0 || len(pytestArgs) > 0 { + req.TestOptions = &testOptions{TestSelection: selection, PytestArgs: pytestArgs} + } + return req +} + +const pytestArgHint = "pass pytest arguments with --pytest-arg, for example --pytest-arg=-k --pytest-arg='orders and not slow'" + +// validatePytestArgs accepts only -k and -m, each with a value in the next argument or attached to it. +func validatePytestArgs(args []string) error { + for i := 0; i < len(args); i++ { + arg := args[i] + switch { + case arg == "-k" || arg == "-m": + if i == len(args)-1 { + return fmt.Errorf("pytest argument %s requires a value: pass it in the next --pytest-arg", arg) + } + i++ + case strings.HasPrefix(arg, "-k") || strings.HasPrefix(arg, "-m"): + case !strings.HasPrefix(arg, "-"): + return fmt.Errorf("unsupported pytest argument %q: pass test paths and test IDs as TEST_SELECTION arguments", arg) + default: + return fmt.Errorf("unsupported pytest argument %q: only -k and -m are supported", arg) + } + } + return nil +} + +// testFlagErrorFunc adds guidance when pytest's -k or -m is passed as a CLI flag. +func testFlagErrorFunc(cmd *cobra.Command, err error) error { + if notExist, ok := errors.AsType[*pflag.NotExistError](err); ok { + if name := notExist.GetSpecifiedName(); name == "k" || name == "m" { + return fmt.Errorf("-%s is a pytest argument: %s", name, pytestArgHint) + } + } + return cmd.Parent().FlagErrorFunc()(cmd, err) +} + +// resolveTestSelections converts relative selection paths with toWorkspacePath and keeps +// the pytest test ID after "::" unchanged. Paths that start with "/" are workspace paths. +func resolveTestSelections(selections []string, toWorkspacePath func(string) (string, error)) ([]string, error) { + var resolved []string + for _, selection := range selections { + p, testID, _ := strings.Cut(selection, "::") + if p == "" { + return nil, fmt.Errorf("test selection %q has no path", selection) + } + if strings.HasPrefix(p, "/") { + p = path.Clean(p) + } else { + var err error + p, err = toWorkspacePath(p) + if err != nil { + return nil, fmt.Errorf("cannot resolve test selection %q: %w", selection, err) + } + } + if strings.Contains(selection, "::") { + p += "::" + testID + } + resolved = append(resolved, p) + } + return resolved, nil +} + +// bundleWorkspacePath maps a local path, relative to the working directory, to its deployed +// path under the bundle's workspace file path. +func bundleWorkspacePath(cwd, syncRoot, workspaceFilePath, localPath string) (string, error) { + if !filepath.IsAbs(localPath) { + localPath = filepath.Join(cwd, localPath) + } + rel, err := filepath.Rel(syncRoot, localPath) + if err != nil || rel == ".." || strings.HasPrefix(rel, ".."+string(filepath.Separator)) { + return "", fmt.Errorf("%s is outside the synchronized bundle files in %s", localPath, syncRoot) + } + return path.Join(workspaceFilePath, filepath.ToSlash(rel)), nil +} + +// rootPathWorkspacePath resolves a relative path against the pipeline's root_path. +func rootPathWorkspacePath(rootPath, relPath string) (string, error) { + if rootPath == "" { + return "", errors.New("the pipeline has no root_path; use an absolute workspace path such as /Workspace/Users/someone@example.com/tests") + } + return path.Join(rootPath, filepath.ToSlash(relPath)), nil +} + +// startTestUpdate starts a test-only update and returns its ID. +func startTestUpdate(ctx context.Context, w *databricks.WorkspaceClient, pipelineID string, req startTestUpdateRequest) (string, error) { + apiClient, err := client.New(w.Config) + if err != nil { + return "", fmt.Errorf("failed to create API client: %w", err) + } + + var response pipelines.StartUpdateResponse + err = apiClient.Do( + ctx, + http.MethodPost, + fmt.Sprintf("/api/2.0/pipelines/%s/updates", pipelineID), + auth.WorkspaceIDHeaders(w.Config), + nil, + req, + &response, + ) + if err != nil { + return "", err + } + return response.UpdateId, nil +} + +// testTarget is the pipeline a test run is submitted to. +type testTarget struct { + key string + pipelineID string + selections []string + w *databricks.WorkspaceClient +} + +func (t testTarget) printHeader(out io.Writer, updateID string) { + if t.key != "" { + fmt.Fprintf(out, "Pipeline: %s (%s)\n", t.key, t.pipelineID) + } else { + fmt.Fprintf(out, "Pipeline: %s\n", t.pipelineID) + } + fmt.Fprintf(out, "Update: %s\n", updateID) +} + +// resolveTestTargetByID identifies the pipeline by --pipeline-id; all positional arguments are selections. +func resolveTestTargetByID(cmd *cobra.Command, args []string, pipelineID string) (testTarget, error) { + if cmd.Flag("target").Value.String() == "" { + cmd.SetContext(root.SkipLoadBundle(cmd.Context())) + } + if err := root.MustWorkspaceClient(cmd, args); err != nil { + return testTarget{}, err + } + ctx := cmd.Context() + w := cmdctx.WorkspaceClient(ctx) + + var rootPath *string + selections, err := resolveTestSelections(args, func(p string) (string, error) { + if rootPath == nil { + pipeline, err := w.Pipelines.Get(ctx, pipelines.GetPipelineRequest{PipelineId: pipelineID}) + if err != nil { + return "", err + } + rootPath = new(string) + if pipeline.Spec != nil { + *rootPath = pipeline.Spec.RootPath + } + } + return rootPathWorkspacePath(*rootPath, p) + }) + if err != nil { + return testTarget{}, err + } + return testTarget{pipelineID: pipelineID, selections: selections, w: w}, nil +} + +// resolveTestTargetInBundle identifies the pipeline by its bundle KEY, which must precede any selections. +func resolveTestTargetInBundle(cmd *cobra.Command, args []string) (testTarget, error) { + var key string + b, err := utils.ProcessBundle(cmd, utils.ProcessOptions{ + PostInitFunc: func(ctx context.Context, b *bundle.Bundle) error { + var err error + key, err = resolvePipelineArgument(ctx, b, args[:min(len(args), 1)]) + return err + }, + ErrorOnEmptyState: true, + SkipInitContext: true, + }) + if err != nil { + return testTarget{}, err + } + ctx := cmd.Context() + suggestPipelineDeploy(ctx, cmd) + + if _, err := resources.Lookup(b, key, isPipeline); err != nil { + return testTarget{}, fmt.Errorf("%w; the first argument must be a pipeline KEY when test selections are given", err) + } + pipelineID, err := resolvePipelineIdFromKey(ctx, b, key) + if err != nil { + return testTarget{}, err + } + + cwd, err := os.Getwd() + if err != nil { + return testTarget{}, err + } + var selectionArgs []string + if len(args) > 1 { + selectionArgs = args[1:] + } + selections, err := resolveTestSelections(selectionArgs, func(p string) (string, error) { + return bundleWorkspacePath(cwd, b.SyncRootPath, b.Config.Workspace.FilePath, p) + }) + if err != nil { + return testTarget{}, err + } + return testTarget{key: key, pipelineID: pipelineID, selections: selections, w: b.WorkspaceClient(ctx)}, nil +} + +func testCommand() *cobra.Command { + cmd := &cobra.Command{ + Use: "test [PIPELINE_KEY] [TEST_SELECTION...] [flags] [--pytest-arg=ARG ...]", + Short: "Run the tests of a pipeline", + Long: `Run the tests of a pipeline in a test-only update. + +PIPELINE_KEY is the unique name of the pipeline, as defined in its YAML file. +It is required when test selections are given. If there is only one pipeline +in the project and no arguments are given, it is selected automatically. +Outside a project, identify the pipeline with --pipeline-id; all positional +arguments are then test selections. + +TEST_SELECTION is a test directory, file, or pytest test ID, for example +tests/test_orders.py::test_backfill. Relative paths are converted to their +deployed workspace paths, or outside a project resolved against the pipeline's +root_path. Paths that start with / are workspace paths. Without a selection, +all of the pipeline's tests run. + +Each --pytest-arg passes one argument to pytest, in the order given. Only -k +and -m are supported, for example: + + databricks pipelines test my_pipeline --pytest-arg=-k --pytest-arg='orders and not slow'`, + } + + var pipelineID string + var pytestArgs []string + + testGroup := cmdgroup.NewFlagGroup("Pipeline Test") + testGroup.FlagSet().StringVar(&pipelineID, "pipeline-id", "", "ID of the pipeline to test, instead of a PIPELINE_KEY.") + testGroup.FlagSet().StringArrayVar(&pytestArgs, "pytest-arg", nil, "Argument to pass to pytest (repeatable). Only -k and -m are supported.") + + wrappedCmd := cmdgroup.NewCommandWithGroupFlag(cmd) + wrappedCmd.AddFlagGroup(testGroup) + + cmd.SetFlagErrorFunc(testFlagErrorFunc) + + cmd.RunE = func(cmd *cobra.Command, args []string) error { + if cmd.ArgsLenAtDash() >= 0 { + return errors.New("the -- separator is not supported: " + pytestArgHint) + } + if root.OutputType(cmd) != flags.OutputText { + return fmt.Errorf("--output %s is not supported by pipelines test", root.OutputType(cmd)) + } + if err := validatePytestArgs(pytestArgs); err != nil { + return err + } + + var target testTarget + var err error + if pipelineID != "" { + target, err = resolveTestTargetByID(cmd, args, pipelineID) + } else { + ctx := logdiag.InitContext(cmd.Context()) + cmd.SetContext(ctx) + if !bundle.HasRoot(ctx) { + return errors.New("outside a project, identify the pipeline with --pipeline-id") + } + target, err = resolveTestTargetInBundle(cmd, args) + } + if err != nil { + return err + } + ctx := cmd.Context() + + req := newStartTestUpdateRequest(target.selections, pytestArgs) + updateID, err := startTestUpdate(ctx, target.w, target.pipelineID, req) + if err != nil { + return err + } + target.printHeader(cmd.OutOrStdout(), updateID) + return nil + } + + return cmd +} diff --git a/cmd/pipelines/test_test.go b/cmd/pipelines/test_test.go new file mode 100644 index 00000000000..1e12fa19094 --- /dev/null +++ b/cmd/pipelines/test_test.go @@ -0,0 +1,116 @@ +package pipelines + +import ( + "encoding/json" + "errors" + "path/filepath" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestValidatePytestArgs(t *testing.T) { + tests := []struct { + args []string + wantErr string + }{ + {args: nil}, + {args: []string{"-k", "orders and not slow", "-m", "smoke,fast"}}, + {args: []string{"-korders", "-msmoke"}}, + {args: []string{"-k", "-m"}}, + {args: []string{"-k"}, wantErr: "pytest argument -k requires a value: pass it in the next --pytest-arg"}, + {args: []string{"-m", "smoke", "-m"}, wantErr: "pytest argument -m requires a value: pass it in the next --pytest-arg"}, + {args: []string{"-x"}, wantErr: `unsupported pytest argument "-x": only -k and -m are supported`}, + {args: []string{"--maxfail=1"}, wantErr: `unsupported pytest argument "--maxfail=1": only -k and -m are supported`}, + {args: []string{"tests/test_orders.py"}, wantErr: `unsupported pytest argument "tests/test_orders.py": pass test paths and test IDs as TEST_SELECTION arguments`}, + } + for _, tc := range tests { + err := validatePytestArgs(tc.args) + if tc.wantErr == "" { + assert.NoError(t, err, tc.args) + } else { + assert.EqualError(t, err, tc.wantErr, tc.args) + } + } +} + +func TestResolveTestSelections(t *testing.T) { + toWorkspacePath := func(p string) (string, error) { + if p == "missing" { + return "", errors.New("not found") + } + return "/Workspace/root/" + p, nil + } + + resolved, err := resolveTestSelections([]string{ + "tests", + "tests/test_orders.py::TestOrders::test_backfill[a::b]", + "/Workspace/Users/someone/../someone/tests/test_x.py::test_x", + }, toWorkspacePath) + require.NoError(t, err) + assert.Equal(t, []string{ + "/Workspace/root/tests", + "/Workspace/root/tests/test_orders.py::TestOrders::test_backfill[a::b]", + "/Workspace/Users/someone/tests/test_x.py::test_x", + }, resolved) + + _, err = resolveTestSelections([]string{"::test_x"}, toWorkspacePath) + assert.EqualError(t, err, `test selection "::test_x" has no path`) + + _, err = resolveTestSelections([]string{"missing::test_x"}, toWorkspacePath) + assert.EqualError(t, err, `cannot resolve test selection "missing::test_x": not found`) +} + +func TestBundleWorkspacePath(t *testing.T) { + syncRoot := t.TempDir() + bundleRoot := filepath.Join(syncRoot, "orders") + filePath := "/Workspace/Users/someone/.bundle/orders/dev/files" + + got, err := bundleWorkspacePath(bundleRoot, syncRoot, filePath, filepath.Join("tests", "test_orders.py")) + require.NoError(t, err) + assert.Equal(t, filePath+"/orders/tests/test_orders.py", got) + + got, err = bundleWorkspacePath(filepath.Join(bundleRoot, "tests"), syncRoot, filePath, filepath.Join("..", "conftest.py")) + require.NoError(t, err) + assert.Equal(t, filePath+"/orders/conftest.py", got) + + got, err = bundleWorkspacePath(syncRoot, syncRoot, filePath, ".") + require.NoError(t, err) + assert.Equal(t, filePath, got) + + _, err = bundleWorkspacePath(syncRoot, syncRoot, filePath, filepath.Join("..", "tests")) + assert.ErrorContains(t, err, "is outside the synchronized bundle files in "+syncRoot) +} + +func TestRootPathWorkspacePath(t *testing.T) { + got, err := rootPathWorkspacePath("/Workspace/Users/someone/orders", "tests/test_orders.py") + require.NoError(t, err) + assert.Equal(t, "/Workspace/Users/someone/orders/tests/test_orders.py", got) + + _, err = rootPathWorkspacePath("", "tests") + assert.EqualError(t, err, "the pipeline has no root_path; use an absolute workspace path such as /Workspace/Users/someone@example.com/tests") +} + +func TestNewStartTestUpdateRequest(t *testing.T) { + tests := []struct { + selection []string + pytestArgs []string + want string + }{ + {want: `{"test_only":true}`}, + { + selection: []string{"/Workspace/tests"}, + want: `{"test_only":true,"test_options":{"test_selection":["/Workspace/tests"]}}`, + }, + { + pytestArgs: []string{"-m", "smoke,fast"}, + want: `{"test_only":true,"test_options":{"pytest_args":["-m","smoke,fast"]}}`, + }, + } + for _, tc := range tests { + body, err := json.Marshal(newStartTestUpdateRequest(tc.selection, tc.pytestArgs)) + require.NoError(t, err) + assert.JSONEq(t, tc.want, string(body)) + } +} From ad9c4beda867f1946e6c8a91aac406c398323060 Mon Sep 17 00:00:00 2001 From: Bhukya Date: Tue, 27 Oct 2026 23:16:22 +0000 Subject: [PATCH 02/10] Test every rejected pytest option and the spaced --pytest-arg form Cover each rejected pytest option with a unit test, and add an acceptance case for `--pytest-arg VALUE`, which keeps commas and spaces. Co-authored-by: Isaac --- .../pipelines/test/selection/output.txt | 26 +++++++++++++++++++ acceptance/pipelines/test/selection/script | 4 +++ cmd/pipelines/test_test.go | 13 ++++++++++ 3 files changed, 43 insertions(+) diff --git a/acceptance/pipelines/test/selection/output.txt b/acceptance/pipelines/test/selection/output.txt index 8b3bca77bc9..46c3137cecc 100644 --- a/acceptance/pipelines/test/selection/output.txt +++ b/acceptance/pipelines/test/selection/output.txt @@ -54,6 +54,32 @@ Update: [UUID] } } +=== --pytest-arg also takes its value as the next argument +>>> [CLI] pipelines test my_pipeline --pytest-arg -k --pytest-arg orders, returns --pytest-arg -m --pytest-arg smoke +Recommendation: This command runs the last deployed version of the code + +If you've made local changes, run 'databricks pipelines deploy' first to ensure they are included. + +Pipeline: my_pipeline ([UUID]) +Update: [UUID] + +>>> print_requests.py //updates +{ + "method": "POST", + "path": "/api/2.0/pipelines/[UUID]/updates", + "body": { + "test_only": true, + "test_options": { + "pytest_args": [ + "-k", + "orders, returns", + "-m", + "smoke" + ] + } + } +} + === Relative paths resolve from the working directory >>> [CLI] pipelines test my_pipeline test_orders.py::test_valid_order Recommendation: This command runs the last deployed version of the code diff --git a/acceptance/pipelines/test/selection/script b/acceptance/pipelines/test/selection/script index 22348d2962f..ca96fb91835 100644 --- a/acceptance/pipelines/test/selection/script +++ b/acceptance/pipelines/test/selection/script @@ -9,6 +9,10 @@ title "Select tests by relative path and test ID, with pytest arguments" trace $CLI pipelines test my_pipeline tests tests/test_orders.py::test_backfill /Workspace/Shared/tests/test_shared.py --pytest-arg=-k --pytest-arg='orders and not slow' --pytest-arg=-msmoke,fast trace print_requests.py //updates +title "--pytest-arg also takes its value as the next argument" +trace $CLI pipelines test my_pipeline --pytest-arg -k --pytest-arg 'orders, returns' --pytest-arg -m --pytest-arg smoke +trace print_requests.py //updates + title "Relative paths resolve from the working directory" cd tests trace $CLI pipelines test my_pipeline test_orders.py::test_valid_order diff --git a/cmd/pipelines/test_test.go b/cmd/pipelines/test_test.go index 1e12fa19094..c887cd10be9 100644 --- a/cmd/pipelines/test_test.go +++ b/cmd/pipelines/test_test.go @@ -35,6 +35,19 @@ func TestValidatePytestArgs(t *testing.T) { } } +// TestValidatePytestArgsRejectsUnsupportedOptions covers every pytest option that the API rejects. +func TestValidatePytestArgsRejectsUnsupportedOptions(t *testing.T) { + for _, arg := range []string{ + "--collect-only", "-x", "--exitfirst", "--maxfail=1", "--lf", "--ff", "--cache-clear", "-n", + "--pdb", "--trace", "-q", "-v", "--color=yes", "--tb=short", "-s", "--capture=no", "-p", + "--junitxml=out.xml", "--html=out.html", "--basetemp=/tmp", "-c", "--rootdir=/", "--confcutdir=/", + "--pyargs", "--timeout=10", "--no-wait", "--junit-xml=out.xml", + } { + err := validatePytestArgs([]string{arg}) + assert.EqualError(t, err, `unsupported pytest argument "`+arg+`": only -k and -m are supported`) + } +} + func TestResolveTestSelections(t *testing.T) { toWorkspacePath := func(p string) (string, error) { if p == "missing" { From a16bd5594e95f8c3d3cbb0ad8969d1b576509be9 Mon Sep 17 00:00:00 2001 From: Bhukya Date: Fri, 30 Oct 2026 10:20:55 +0000 Subject: [PATCH 03/10] Resolve pipelines test selections like the test IDs the service reports The service names a test case relative to pytest's rootdir: the deepest folder that holds the pipeline's root_path and every test source, or, without a root_path, the folder of every test source. A test ID copied from the results now runs that test again in any project layout. A relative selection that exists locally still maps to its deployed path. Any other relative selection resolves against that folder, which the command reads from the deployed pipeline. With --pipeline-id, all relative selections resolve against it. Co-authored-by: Isaac --- .../pipelines/test/arguments/output.txt | 9 +- .../pipelines/test/pipeline-id/output.txt | 2 +- .../pipelines/test/test-ids/databricks.yml | 11 ++ .../pipelines/test/test-ids/out.test.toml | 1 + acceptance/pipelines/test/test-ids/output.txt | 46 ++++++ .../test/test-ids/pipeline/tests/test_a.py | 2 + .../pipeline/transformations/orders.py | 1 + acceptance/pipelines/test/test-ids/script | 16 ++ cmd/pipelines/test.go | 150 +++++++++++++++--- cmd/pipelines/test_test.go | 85 +++++++++- 10 files changed, 292 insertions(+), 31 deletions(-) create mode 100644 acceptance/pipelines/test/test-ids/databricks.yml create mode 100644 acceptance/pipelines/test/test-ids/out.test.toml create mode 100644 acceptance/pipelines/test/test-ids/output.txt create mode 100644 acceptance/pipelines/test/test-ids/pipeline/tests/test_a.py create mode 100644 acceptance/pipelines/test/test-ids/pipeline/transformations/orders.py create mode 100644 acceptance/pipelines/test/test-ids/script diff --git a/acceptance/pipelines/test/arguments/output.txt b/acceptance/pipelines/test/arguments/output.txt index dc32fe74d13..4c3eed3f7fd 100644 --- a/acceptance/pipelines/test/arguments/output.txt +++ b/acceptance/pipelines/test/arguments/output.txt @@ -9,10 +9,11 @@ Outside a project, identify the pipeline with --pipeline-id; all positional arguments are then test selections. TEST_SELECTION is a test directory, file, or pytest test ID, for example -tests/test_orders.py::test_backfill. Relative paths are converted to their -deployed workspace paths, or outside a project resolved against the pipeline's -root_path. Paths that start with / are workspace paths. Without a selection, -all of the pipeline's tests run. +tests/test_orders.py::test_backfill. A relative path that exists locally is +converted to its deployed workspace path. Any other relative path is a test ID +as the results show it: relative to the deepest folder that holds the +pipeline's root_path and its test sources. Paths that start with / are +workspace paths. Without a selection, all of the pipeline's tests run. Each --pytest-arg passes one argument to pytest, in the order given. Only -k and -m are supported, for example: diff --git a/acceptance/pipelines/test/pipeline-id/output.txt b/acceptance/pipelines/test/pipeline-id/output.txt index 9138ee2e806..2952c195601 100644 --- a/acceptance/pipelines/test/pipeline-id/output.txt +++ b/acceptance/pipelines/test/pipeline-id/output.txt @@ -21,7 +21,7 @@ Update: [UUID] === Without a root_path, test selections must be workspace paths >>> musterr [CLI] pipelines test --pipeline-id [UUID] tests -Error: cannot resolve test selection "tests": the pipeline has no root_path; use an absolute workspace path such as /Workspace/Users/someone@example.com/tests +Error: cannot resolve test selection "tests": the pipeline has no root_path or testing.sources; use an absolute workspace path such as /Workspace/Users/someone@example.com/tests >>> [CLI] pipelines test --pipeline-id [UUID] /Workspace/Shared/tests/test_shared.py::test_x Pipeline: [UUID] diff --git a/acceptance/pipelines/test/test-ids/databricks.yml b/acceptance/pipelines/test/test-ids/databricks.yml new file mode 100644 index 00000000000..423404790c4 --- /dev/null +++ b/acceptance/pipelines/test/test-ids/databricks.yml @@ -0,0 +1,11 @@ +bundle: + name: test-pipeline-test-ids + +resources: + pipelines: + my_pipeline: + name: test-pipeline-test-ids + root_path: ./pipeline + libraries: + - file: + path: ./pipeline/transformations/orders.py diff --git a/acceptance/pipelines/test/test-ids/out.test.toml b/acceptance/pipelines/test/test-ids/out.test.toml new file mode 100644 index 00000000000..18b1a88417e --- /dev/null +++ b/acceptance/pipelines/test/test-ids/out.test.toml @@ -0,0 +1 @@ +Cloud = false diff --git a/acceptance/pipelines/test/test-ids/output.txt b/acceptance/pipelines/test/test-ids/output.txt new file mode 100644 index 00000000000..f257359806e --- /dev/null +++ b/acceptance/pipelines/test/test-ids/output.txt @@ -0,0 +1,46 @@ + +>>> [CLI] pipelines deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-pipeline-test-ids/default/files... +Created pipelines.my_pipeline +Files: 3 uploaded, 0 deleted +Resources: 1 created, 0 changed, 0 deleted, 0 unchanged +View your pipeline my_pipeline here: [DATABRICKS_URL]/pipelines/[UUID]?w=[WORKSPACE_ID] + +=== A test ID resolves against the folder that test IDs are relative to + +>>> [CLI] pipelines test my_pipeline tests/test_a.py::test_one +Recommendation: This command runs the last deployed version of the code + +If you've made local changes, run 'databricks pipelines deploy' first to ensure they are included. + +Pipeline: my_pipeline ([UUID]) +Update: [UUID] + +>>> print_requests.py //updates +{"test_selection":["/Workspace/Users/[USERNAME]/.bundle/test-pipeline-test-ids/default/files/pipeline/tests/test_a.py::test_one"]} + +=== The same ID from the pipeline folder, where it is a local path + +>>> [CLI] pipelines test my_pipeline tests/test_a.py::test_one +Recommendation: This command runs the last deployed version of the code + +If you've made local changes, run 'databricks pipelines deploy' first to ensure they are included. + +Pipeline: my_pipeline ([UUID]) +Update: [UUID] + +>>> print_requests.py //updates +{"test_selection":["/Workspace/Users/[USERNAME]/.bundle/test-pipeline-test-ids/default/files/pipeline/tests/test_a.py::test_one"]} + +=== A local path from the bundle root + +>>> [CLI] pipelines test my_pipeline pipeline/tests/test_a.py +Recommendation: This command runs the last deployed version of the code + +If you've made local changes, run 'databricks pipelines deploy' first to ensure they are included. + +Pipeline: my_pipeline ([UUID]) +Update: [UUID] + +>>> print_requests.py //updates +{"test_selection":["/Workspace/Users/[USERNAME]/.bundle/test-pipeline-test-ids/default/files/pipeline/tests/test_a.py"]} diff --git a/acceptance/pipelines/test/test-ids/pipeline/tests/test_a.py b/acceptance/pipelines/test/test-ids/pipeline/tests/test_a.py new file mode 100644 index 00000000000..65de24b9c57 --- /dev/null +++ b/acceptance/pipelines/test/test-ids/pipeline/tests/test_a.py @@ -0,0 +1,2 @@ +def test_one(): + pass diff --git a/acceptance/pipelines/test/test-ids/pipeline/transformations/orders.py b/acceptance/pipelines/test/test-ids/pipeline/transformations/orders.py new file mode 100644 index 00000000000..7d4290a117a --- /dev/null +++ b/acceptance/pipelines/test/test-ids/pipeline/transformations/orders.py @@ -0,0 +1 @@ +x = 1 diff --git a/acceptance/pipelines/test/test-ids/script b/acceptance/pipelines/test/test-ids/script new file mode 100644 index 00000000000..3972d69e926 --- /dev/null +++ b/acceptance/pipelines/test/test-ids/script @@ -0,0 +1,16 @@ +trace $CLI pipelines deploy +rm "$OUT_REQUESTS" + +title "A test ID resolves against the folder that test IDs are relative to\n" +trace $CLI pipelines test my_pipeline tests/test_a.py::test_one +trace print_requests.py //updates | jq -c .body.test_options + +title "The same ID from the pipeline folder, where it is a local path\n" +cd pipeline +trace $CLI pipelines test my_pipeline tests/test_a.py::test_one +cd .. +trace print_requests.py //updates | jq -c .body.test_options + +title "A local path from the bundle root\n" +trace $CLI pipelines test my_pipeline pipeline/tests/test_a.py +trace print_requests.py //updates | jq -c .body.test_options diff --git a/cmd/pipelines/test.go b/cmd/pipelines/test.go index 7a9da8bef9d..b78c55ad057 100644 --- a/cmd/pipelines/test.go +++ b/cmd/pipelines/test.go @@ -10,6 +10,7 @@ import ( "path" "path/filepath" "strings" + "sync" "github.com/databricks/cli/bundle" "github.com/databricks/cli/bundle/resources" @@ -118,12 +119,113 @@ func bundleWorkspacePath(cwd, syncRoot, workspaceFilePath, localPath string) (st return path.Join(workspaceFilePath, filepath.ToSlash(rel)), nil } -// rootPathWorkspacePath resolves a relative path against the pipeline's root_path. -func rootPathWorkspacePath(rootPath, relPath string) (string, error) { - if rootPath == "" { - return "", errors.New("the pipeline has no root_path; use an absolute workspace path such as /Workspace/Users/someone@example.com/tests") +// bundleTestSelectionPath maps a relative selection to a workspace path. A path that exists locally, +// relative to the working directory, maps to its deployed path. Otherwise the selection is a test +// ID that the CLI printed, relative to the folder that testIDBase returns, unless it leaves that folder. +func bundleTestSelectionPath(cwd, syncRoot, workspaceFilePath string, testIDBase func() (string, error), localPath string) (string, error) { + rel := path.Clean(filepath.ToSlash(localPath)) + leavesBase := rel == ".." || strings.HasPrefix(rel, "../") + if _, err := os.Stat(filepath.Join(cwd, localPath)); err != nil && !leavesBase { + base, err := testIDBase() + if err != nil { + return "", err + } + if base != "" { + return testIDBasePath(base, localPath) + } + } + return bundleWorkspacePath(cwd, syncRoot, workspaceFilePath, localPath) +} + +// testIDBase returns the folder that the service names test cases relative to, or "" if the +// pipeline has neither a root_path nor test sources. pytest runs with this folder as its rootdir: +// the deepest folder that holds the root path and every test source or, without a root path, the +// folder of every test source. +func testIDBase(rootPath string, sources []string) string { + var folders [][]string + if rootPath != "" { + folders = append(folders, pathSegments(normalizeWorkspacePath(rootPath))) + } + for _, source := range sources { + segments := pathSegments(normalizeWorkspacePath(strings.TrimSuffix(source, "/**"))) + if rootPath == "" && len(segments) > 0 { + segments = segments[:len(segments)-1] + } + folders = append(folders, segments) + } + if len(folders) == 0 { + return "" + } + common := folders[0] + for _, folder := range folders[1:] { + n := 0 + for n < len(common) && n < len(folder) && common[n] == folder[n] { + n++ + } + common = common[:n] + } + return "/" + strings.Join(common, "/") +} + +// normalizeWorkspacePath adds the /Workspace prefix to an absolute path without it, as the service does. +func normalizeWorkspacePath(p string) string { + if strings.HasPrefix(p, "/") && !strings.HasPrefix(p, "/Workspace") { + return "/Workspace" + p + } + return p +} + +func pathSegments(p string) []string { + var segments []string + for segment := range strings.SplitSeq(p, "/") { + if segment != "" { + segments = append(segments, segment) + } + } + return segments +} + +// testIDBasePath resolves a relative path against the folder that test IDs are relative to. +func testIDBasePath(base, relPath string) (string, error) { + if base == "" { + return "", errors.New("the pipeline has no root_path or testing.sources; use an absolute workspace path such as /Workspace/Users/someone@example.com/tests") } - return path.Join(rootPath, filepath.ToSlash(relPath)), nil + return path.Join(base, filepath.ToSlash(relPath)), nil +} + +// pipelineTestPaths holds the fields of a Get Pipeline response that decide the folder of test IDs. +// The Go SDK has no spec.testing yet, so the response is decoded from the raw API call. +type pipelineTestPaths struct { + Spec struct { + RootPath string `json:"root_path"` + Testing struct { + Sources []struct { + Glob struct { + Include string `json:"include"` + } `json:"glob"` + } `json:"sources"` + } `json:"testing"` + } `json:"spec"` +} + +// getTestIDBase reads the deployed pipeline and returns the folder that its test IDs are relative to. +func getTestIDBase(ctx context.Context, w *databricks.WorkspaceClient, pipelineID string) (string, error) { + apiClient, err := client.New(w.Config) + if err != nil { + return "", fmt.Errorf("failed to create API client: %w", err) + } + var response pipelineTestPaths + err = apiClient.Do(ctx, http.MethodGet, "/api/2.0/pipelines/"+pipelineID, auth.WorkspaceIDHeaders(w.Config), nil, nil, &response) + if err != nil { + return "", err + } + var sources []string + for _, source := range response.Spec.Testing.Sources { + if source.Glob.Include != "" { + sources = append(sources, source.Glob.Include) + } + } + return testIDBase(response.Spec.RootPath, sources), nil } // startTestUpdate starts a test-only update and returns its ID. @@ -166,6 +268,13 @@ func (t testTarget) printHeader(out io.Writer, updateID string) { fmt.Fprintf(out, "Update: %s\n", updateID) } +// lazyTestIDBase returns a function that reads the folder of test IDs once, when a selection needs it. +func lazyTestIDBase(ctx context.Context, w *databricks.WorkspaceClient, pipelineID string) func() (string, error) { + return sync.OnceValues(func() (string, error) { + return getTestIDBase(ctx, w, pipelineID) + }) +} + // resolveTestTargetByID identifies the pipeline by --pipeline-id; all positional arguments are selections. func resolveTestTargetByID(cmd *cobra.Command, args []string, pipelineID string) (testTarget, error) { if cmd.Flag("target").Value.String() == "" { @@ -177,19 +286,13 @@ func resolveTestTargetByID(cmd *cobra.Command, args []string, pipelineID string) ctx := cmd.Context() w := cmdctx.WorkspaceClient(ctx) - var rootPath *string + base := lazyTestIDBase(ctx, w, pipelineID) selections, err := resolveTestSelections(args, func(p string) (string, error) { - if rootPath == nil { - pipeline, err := w.Pipelines.Get(ctx, pipelines.GetPipelineRequest{PipelineId: pipelineID}) - if err != nil { - return "", err - } - rootPath = new(string) - if pipeline.Spec != nil { - *rootPath = pipeline.Spec.RootPath - } + base, err := base() + if err != nil { + return "", err } - return rootPathWorkspacePath(*rootPath, p) + return testIDBasePath(base, p) }) if err != nil { return testTarget{}, err @@ -231,13 +334,15 @@ func resolveTestTargetInBundle(cmd *cobra.Command, args []string) (testTarget, e if len(args) > 1 { selectionArgs = args[1:] } + w := b.WorkspaceClient(ctx) + base := lazyTestIDBase(ctx, w, pipelineID) selections, err := resolveTestSelections(selectionArgs, func(p string) (string, error) { - return bundleWorkspacePath(cwd, b.SyncRootPath, b.Config.Workspace.FilePath, p) + return bundleTestSelectionPath(cwd, b.SyncRootPath, b.Config.Workspace.FilePath, base, p) }) if err != nil { return testTarget{}, err } - return testTarget{key: key, pipelineID: pipelineID, selections: selections, w: b.WorkspaceClient(ctx)}, nil + return testTarget{key: key, pipelineID: pipelineID, selections: selections, w: w}, nil } func testCommand() *cobra.Command { @@ -253,10 +358,11 @@ Outside a project, identify the pipeline with --pipeline-id; all positional arguments are then test selections. TEST_SELECTION is a test directory, file, or pytest test ID, for example -tests/test_orders.py::test_backfill. Relative paths are converted to their -deployed workspace paths, or outside a project resolved against the pipeline's -root_path. Paths that start with / are workspace paths. Without a selection, -all of the pipeline's tests run. +tests/test_orders.py::test_backfill. A relative path that exists locally is +converted to its deployed workspace path. Any other relative path is a test ID +as the results show it: relative to the deepest folder that holds the +pipeline's root_path and its test sources. Paths that start with / are +workspace paths. Without a selection, all of the pipeline's tests run. Each --pytest-arg passes one argument to pytest, in the order given. Only -k and -m are supported, for example: diff --git a/cmd/pipelines/test_test.go b/cmd/pipelines/test_test.go index c887cd10be9..3c73fade96c 100644 --- a/cmd/pipelines/test_test.go +++ b/cmd/pipelines/test_test.go @@ -3,6 +3,7 @@ package pipelines import ( "encoding/json" "errors" + "os" "path/filepath" "testing" @@ -96,13 +97,89 @@ func TestBundleWorkspacePath(t *testing.T) { assert.ErrorContains(t, err, "is outside the synchronized bundle files in "+syncRoot) } -func TestRootPathWorkspacePath(t *testing.T) { - got, err := rootPathWorkspacePath("/Workspace/Users/someone/orders", "tests/test_orders.py") +func TestTestIDBase(t *testing.T) { + tests := []struct { + name string + rootPath string + sources []string + want string + }{ + { + name: "deepest folder holding the root path and every test source", + rootPath: "/Workspace/Users/u/files/src/my_pipeline", + sources: []string{"/Workspace/Users/u/files/tests/**", "/Workspace/Users/u/files/checks/test_x.py"}, + want: "/Workspace/Users/u/files", + }, + { + name: "root path that holds the test sources", + rootPath: "/Users/u/sl", + sources: []string{"/Users/u/sl/tests/**"}, + want: "/Workspace/Users/u/sl", + }, + { + name: "without a root path, the folder of a file test source", + sources: []string{"/Workspace/Users/u/tests/test_x.py"}, + want: "/Workspace/Users/u/tests", + }, + { + name: "without a root path, the deepest folder holding the folders of every test source", + sources: []string{"/Workspace/Users/u/a/tests/**", "/Workspace/Users/u/b/tests/**"}, + want: "/Workspace/Users/u", + }, + { + name: "root path without test sources", + rootPath: "/Workspace/Users/u/orders", + want: "/Workspace/Users/u/orders", + }, + { + name: "neither", + want: "", + }, + } + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + assert.Equal(t, tc.want, testIDBase(tc.rootPath, tc.sources)) + }) + } +} + +func TestTestIDBasePath(t *testing.T) { + got, err := testIDBasePath("/Workspace/Users/someone/orders", "tests/test_orders.py") require.NoError(t, err) assert.Equal(t, "/Workspace/Users/someone/orders/tests/test_orders.py", got) - _, err = rootPathWorkspacePath("", "tests") - assert.EqualError(t, err, "the pipeline has no root_path; use an absolute workspace path such as /Workspace/Users/someone@example.com/tests") + _, err = testIDBasePath("", "tests") + assert.EqualError(t, err, "the pipeline has no root_path or testing.sources; use an absolute workspace path such as /Workspace/Users/someone@example.com/tests") +} + +func TestBundleTestSelectionPath(t *testing.T) { + syncRoot := t.TempDir() + require.NoError(t, os.MkdirAll(filepath.Join(syncRoot, "pipeline", "tests"), 0o755)) + require.NoError(t, os.WriteFile(filepath.Join(syncRoot, "pipeline", "tests", "test_a.py"), nil, 0o644)) + filePath := "/Workspace/Users/someone/.bundle/orders/dev/files" + base := func() (string, error) { return filePath + "/pipeline", nil } + noBase := func() (string, error) { return "", nil } + + // A path that exists where the user stands is a local path. + got, err := bundleTestSelectionPath(filepath.Join(syncRoot, "pipeline"), syncRoot, filePath, base, "tests/test_a.py") + require.NoError(t, err) + assert.Equal(t, filePath+"/pipeline/tests/test_a.py", got) + + // Otherwise it is a test ID, relative to the folder that test IDs are relative to. + got, err = bundleTestSelectionPath(syncRoot, syncRoot, filePath, base, "tests/test_a.py") + require.NoError(t, err) + assert.Equal(t, filePath+"/pipeline/tests/test_a.py", got) + + // Without that folder, it is a local path. + got, err = bundleTestSelectionPath(syncRoot, syncRoot, filePath, noBase, "tests/test_a.py") + require.NoError(t, err) + assert.Equal(t, filePath+"/tests/test_a.py", got) + + // A path that leaves that folder is a local path too. + for _, testIDBase := range []func() (string, error){base, noBase} { + _, err = bundleTestSelectionPath(syncRoot, syncRoot, filePath, testIDBase, filepath.Join("..", "tests")) + assert.ErrorContains(t, err, "is outside the synchronized bundle files in "+syncRoot) + } } func TestNewStartTestUpdateRequest(t *testing.T) { From 31b5ec31db450cdb47c6a8ff0865670ea0b19344 Mon Sep 17 00:00:00 2001 From: Bhukya Date: Fri, 30 Oct 2026 15:06:32 +0000 Subject: [PATCH 04/10] Map local pipelines test paths like TranslatePaths and mark the command preview A local test path mapped to workspace.file_path in every sync mode. A source-linked deployment runs the source files, so a local path now maps under the sync root, as TranslatePaths maps resource paths. An immutable deployment runs a snapshot whose path the command does not know, so a local path is an error there that points to test IDs and workspace paths, which still work. The folder of test IDs is read once, and only when a selection is a relative path. Paths in errors use forward slashes. The Short text starts with *Preview*, like other preview commands. Tests: the option list covers each rejected form once, and a folder test source without root_path has its parent as the folder of test IDs, as the service computes it. Co-authored-by: Isaac --- .../pipelines/databricks-cli-help/output.txt | 2 +- cmd/pipelines/test.go | 85 ++++++++++++------- cmd/pipelines/test_test.go | 75 +++++++++++----- 3 files changed, 111 insertions(+), 51 deletions(-) diff --git a/acceptance/pipelines/databricks-cli-help/output.txt b/acceptance/pipelines/databricks-cli-help/output.txt index 3fb01646d00..adc729f4f5c 100644 --- a/acceptance/pipelines/databricks-cli-help/output.txt +++ b/acceptance/pipelines/databricks-cli-help/output.txt @@ -30,7 +30,7 @@ Main Commands run Run a pipeline show Preview a table's columns and sample rows stop Stop a pipeline - test Run the tests of a pipeline + test *Preview* Run the tests of a pipeline Management Commands apply-environment *Public Preview* Apply the latest environment to the pipeline. diff --git a/cmd/pipelines/test.go b/cmd/pipelines/test.go index b78c55ad057..373ebab8be4 100644 --- a/cmd/pipelines/test.go +++ b/cmd/pipelines/test.go @@ -10,9 +10,9 @@ import ( "path" "path/filepath" "strings" - "sync" "github.com/databricks/cli/bundle" + "github.com/databricks/cli/bundle/config" "github.com/databricks/cli/bundle/resources" "github.com/databricks/cli/cmd/bundle/utils" "github.com/databricks/cli/cmd/root" @@ -107,34 +107,55 @@ func resolveTestSelections(selections []string, toWorkspacePath func(string) (st } // bundleWorkspacePath maps a local path, relative to the working directory, to its deployed -// path under the bundle's workspace file path. -func bundleWorkspacePath(cwd, syncRoot, workspaceFilePath, localPath string) (string, error) { +// path under filesRoot, the deployed path of the bundle's synchronized files. +func bundleWorkspacePath(cwd, syncRoot, filesRoot, localPath string) (string, error) { if !filepath.IsAbs(localPath) { localPath = filepath.Join(cwd, localPath) } rel, err := filepath.Rel(syncRoot, localPath) if err != nil || rel == ".." || strings.HasPrefix(rel, ".."+string(filepath.Separator)) { - return "", fmt.Errorf("%s is outside the synchronized bundle files in %s", localPath, syncRoot) + return "", fmt.Errorf("%s is outside the synchronized bundle files in %s", filepath.ToSlash(localPath), filepath.ToSlash(syncRoot)) + } + return path.Join(filesRoot, filepath.ToSlash(rel)), nil +} + +// bundleFilesRoot returns the deployed path of the bundle's synchronized files, as TranslatePaths +// maps resource paths, or "" for an immutable deployment, whose snapshot path the command does not know. +func bundleFilesRoot(b *bundle.Bundle) string { + switch { + case b.SyncMode() == config.SyncModeImmutable: + return "" + case b.IsSourceLinkedDeployment(): + return b.SyncRootPath + default: + return b.Config.Workspace.FilePath } - return path.Join(workspaceFilePath, filepath.ToSlash(rel)), nil } // bundleTestSelectionPath maps a relative selection to a workspace path. A path that exists locally, -// relative to the working directory, maps to its deployed path. Otherwise the selection is a test -// ID that the CLI printed, relative to the folder that testIDBase returns, unless it leaves that folder. -func bundleTestSelectionPath(cwd, syncRoot, workspaceFilePath string, testIDBase func() (string, error), localPath string) (string, error) { +// relative to the working directory, maps to its deployed path under filesRoot. Otherwise the +// selection is a test ID that the CLI printed, relative to testIDBase, unless it leaves the working +// directory. +func bundleTestSelectionPath(cwd, syncRoot, filesRoot, testIDBase, localPath string) (string, error) { rel := path.Clean(filepath.ToSlash(localPath)) - leavesBase := rel == ".." || strings.HasPrefix(rel, "../") - if _, err := os.Stat(filepath.Join(cwd, localPath)); err != nil && !leavesBase { - base, err := testIDBase() - if err != nil { - return "", err - } - if base != "" { - return testIDBasePath(base, localPath) + leavesCwd := rel == ".." || strings.HasPrefix(rel, "../") + if _, err := os.Stat(filepath.Join(cwd, localPath)); err != nil && !leavesCwd && testIDBase != "" { + return testIDBasePath(testIDBase, localPath) + } + if filesRoot == "" { + return "", errors.New("local test paths are not supported with workspace.sync_mode immutable; pass the test ID that the results show or an absolute workspace path") + } + return bundleWorkspacePath(cwd, syncRoot, filesRoot, localPath) +} + +// needsTestIDBase reports whether a selection is a relative path, which may be a test ID. +func needsTestIDBase(selections []string) bool { + for _, selection := range selections { + if p, _, _ := strings.Cut(selection, "::"); !strings.HasPrefix(p, "/") { + return true } } - return bundleWorkspacePath(cwd, syncRoot, workspaceFilePath, localPath) + return false } // testIDBase returns the folder that the service names test cases relative to, or "" if the @@ -268,11 +289,12 @@ func (t testTarget) printHeader(out io.Writer, updateID string) { fmt.Fprintf(out, "Update: %s\n", updateID) } -// lazyTestIDBase returns a function that reads the folder of test IDs once, when a selection needs it. -func lazyTestIDBase(ctx context.Context, w *databricks.WorkspaceClient, pipelineID string) func() (string, error) { - return sync.OnceValues(func() (string, error) { - return getTestIDBase(ctx, w, pipelineID) - }) +// readTestIDBase reads the folder of test IDs from the deployed pipeline, if a selection needs it. +func readTestIDBase(ctx context.Context, w *databricks.WorkspaceClient, pipelineID string, selections []string) (string, error) { + if !needsTestIDBase(selections) { + return "", nil + } + return getTestIDBase(ctx, w, pipelineID) } // resolveTestTargetByID identifies the pipeline by --pipeline-id; all positional arguments are selections. @@ -286,12 +308,11 @@ func resolveTestTargetByID(cmd *cobra.Command, args []string, pipelineID string) ctx := cmd.Context() w := cmdctx.WorkspaceClient(ctx) - base := lazyTestIDBase(ctx, w, pipelineID) + base, err := readTestIDBase(ctx, w, pipelineID, args) + if err != nil { + return testTarget{}, err + } selections, err := resolveTestSelections(args, func(p string) (string, error) { - base, err := base() - if err != nil { - return "", err - } return testIDBasePath(base, p) }) if err != nil { @@ -335,9 +356,13 @@ func resolveTestTargetInBundle(cmd *cobra.Command, args []string) (testTarget, e selectionArgs = args[1:] } w := b.WorkspaceClient(ctx) - base := lazyTestIDBase(ctx, w, pipelineID) + base, err := readTestIDBase(ctx, w, pipelineID, selectionArgs) + if err != nil { + return testTarget{}, err + } + filesRoot := bundleFilesRoot(b) selections, err := resolveTestSelections(selectionArgs, func(p string) (string, error) { - return bundleTestSelectionPath(cwd, b.SyncRootPath, b.Config.Workspace.FilePath, base, p) + return bundleTestSelectionPath(cwd, b.SyncRootPath, filesRoot, base, p) }) if err != nil { return testTarget{}, err @@ -348,7 +373,7 @@ func resolveTestTargetInBundle(cmd *cobra.Command, args []string) (testTarget, e func testCommand() *cobra.Command { cmd := &cobra.Command{ Use: "test [PIPELINE_KEY] [TEST_SELECTION...] [flags] [--pytest-arg=ARG ...]", - Short: "Run the tests of a pipeline", + Short: "*Preview* Run the tests of a pipeline", Long: `Run the tests of a pipeline in a test-only update. PIPELINE_KEY is the unique name of the pipeline, as defined in its YAML file. diff --git a/cmd/pipelines/test_test.go b/cmd/pipelines/test_test.go index 3c73fade96c..fc67a1b879f 100644 --- a/cmd/pipelines/test_test.go +++ b/cmd/pipelines/test_test.go @@ -7,6 +7,9 @@ import ( "path/filepath" "testing" + "github.com/databricks/cli/bundle" + "github.com/databricks/cli/bundle/config" + "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" ) @@ -36,14 +39,9 @@ func TestValidatePytestArgs(t *testing.T) { } } -// TestValidatePytestArgsRejectsUnsupportedOptions covers every pytest option that the API rejects. +// TestValidatePytestArgsRejectsUnsupportedOptions covers each form of option that the API rejects. func TestValidatePytestArgsRejectsUnsupportedOptions(t *testing.T) { - for _, arg := range []string{ - "--collect-only", "-x", "--exitfirst", "--maxfail=1", "--lf", "--ff", "--cache-clear", "-n", - "--pdb", "--trace", "-q", "-v", "--color=yes", "--tb=short", "-s", "--capture=no", "-p", - "--junitxml=out.xml", "--html=out.html", "--basetemp=/tmp", "-c", "--rootdir=/", "--confcutdir=/", - "--pyargs", "--timeout=10", "--no-wait", "--junit-xml=out.xml", - } { + for _, arg := range []string{"--collect-only", "-n", "--rootdir=/", "--junit-xml=out.xml"} { err := validatePytestArgs([]string{arg}) assert.EqualError(t, err, `unsupported pytest argument "`+arg+`": only -k and -m are supported`) } @@ -126,6 +124,11 @@ func TestTestIDBase(t *testing.T) { sources: []string{"/Workspace/Users/u/a/tests/**", "/Workspace/Users/u/b/tests/**"}, want: "/Workspace/Users/u", }, + { + name: "without a root path, the parent of a folder test source", + sources: []string{"/Workspace/Users/u/tests/**"}, + want: "/Workspace/Users/u", + }, { name: "root path without test sources", rootPath: "/Workspace/Users/u/orders", @@ -152,34 +155,66 @@ func TestTestIDBasePath(t *testing.T) { assert.EqualError(t, err, "the pipeline has no root_path or testing.sources; use an absolute workspace path such as /Workspace/Users/someone@example.com/tests") } +func TestBundleFilesRoot(t *testing.T) { + filePath := "/Workspace/Users/someone/.bundle/orders/dev/files" + syncRoot := "/Workspace/Users/someone/orders" + for _, tc := range []struct { + mode config.SyncMode + want string + }{ + {mode: config.SyncModeOverwrite, want: filePath}, + // Source-linked deployments run the source files, as TranslatePaths maps them. + {mode: config.SyncModeInPlace, want: syncRoot}, + // Immutable deployments run a snapshot, whose path the command does not know. + {mode: config.SyncModeImmutable, want: ""}, + } { + b := &bundle.Bundle{SyncRootPath: syncRoot} + b.Config.Workspace.FilePath = filePath + b.Config.Workspace.SyncMode = tc.mode + assert.Equal(t, tc.want, bundleFilesRoot(b), tc.mode) + } +} + func TestBundleTestSelectionPath(t *testing.T) { syncRoot := t.TempDir() require.NoError(t, os.MkdirAll(filepath.Join(syncRoot, "pipeline", "tests"), 0o755)) require.NoError(t, os.WriteFile(filepath.Join(syncRoot, "pipeline", "tests", "test_a.py"), nil, 0o644)) - filePath := "/Workspace/Users/someone/.bundle/orders/dev/files" - base := func() (string, error) { return filePath + "/pipeline", nil } - noBase := func() (string, error) { return "", nil } + filesRoot := "/Workspace/Users/someone/.bundle/orders/dev/files" + base := filesRoot + "/pipeline" // A path that exists where the user stands is a local path. - got, err := bundleTestSelectionPath(filepath.Join(syncRoot, "pipeline"), syncRoot, filePath, base, "tests/test_a.py") + got, err := bundleTestSelectionPath(filepath.Join(syncRoot, "pipeline"), syncRoot, filesRoot, base, "tests/test_a.py") require.NoError(t, err) - assert.Equal(t, filePath+"/pipeline/tests/test_a.py", got) + assert.Equal(t, filesRoot+"/pipeline/tests/test_a.py", got) // Otherwise it is a test ID, relative to the folder that test IDs are relative to. - got, err = bundleTestSelectionPath(syncRoot, syncRoot, filePath, base, "tests/test_a.py") + got, err = bundleTestSelectionPath(syncRoot, syncRoot, filesRoot, base, "tests/test_a.py") require.NoError(t, err) - assert.Equal(t, filePath+"/pipeline/tests/test_a.py", got) + assert.Equal(t, filesRoot+"/pipeline/tests/test_a.py", got) // Without that folder, it is a local path. - got, err = bundleTestSelectionPath(syncRoot, syncRoot, filePath, noBase, "tests/test_a.py") + got, err = bundleTestSelectionPath(syncRoot, syncRoot, filesRoot, "", "tests/test_a.py") require.NoError(t, err) - assert.Equal(t, filePath+"/tests/test_a.py", got) + assert.Equal(t, filesRoot+"/tests/test_a.py", got) - // A path that leaves that folder is a local path too. - for _, testIDBase := range []func() (string, error){base, noBase} { - _, err = bundleTestSelectionPath(syncRoot, syncRoot, filePath, testIDBase, filepath.Join("..", "tests")) - assert.ErrorContains(t, err, "is outside the synchronized bundle files in "+syncRoot) + // A path that leaves that folder is a local path too. The error shows forward slashes. + for _, testIDBase := range []string{base, ""} { + _, err = bundleTestSelectionPath(syncRoot, syncRoot, filesRoot, testIDBase, filepath.Join("..", "tests")) + assert.ErrorContains(t, err, "is outside the synchronized bundle files in "+filepath.ToSlash(syncRoot)) } + + // When the deployed path of local files is unknown, a local path is an error, but a test ID still works. + _, err = bundleTestSelectionPath(filepath.Join(syncRoot, "pipeline"), syncRoot, "", base, "tests/test_a.py") + assert.EqualError(t, err, "local test paths are not supported with workspace.sync_mode immutable; pass the test ID that the results show or an absolute workspace path") + got, err = bundleTestSelectionPath(syncRoot, syncRoot, "", base, "tests/test_a.py") + require.NoError(t, err) + assert.Equal(t, filesRoot+"/pipeline/tests/test_a.py", got) +} + +func TestNeedsTestIDBase(t *testing.T) { + assert.False(t, needsTestIDBase(nil)) + assert.False(t, needsTestIDBase([]string{"/Workspace/tests", "/Workspace/a.py::test_x"})) + assert.True(t, needsTestIDBase([]string{"/Workspace/tests", "tests/test_a.py::test_x"})) } func TestNewStartTestUpdateRequest(t *testing.T) { From f7898c2b3e3add82ff37a6b5e2ba6c50aa60e48b Mon Sep 17 00:00:00 2001 From: Bhukya Date: Fri, 30 Oct 2026 18:59:04 +0000 Subject: [PATCH 05/10] Link the changelog entry to its PR Co-authored-by: Isaac --- .nextchanges/cli/pipelines-test.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.nextchanges/cli/pipelines-test.md b/.nextchanges/cli/pipelines-test.md index f7ac2beee9b..4d548dd6581 100644 --- a/.nextchanges/cli/pipelines-test.md +++ b/.nextchanges/cli/pipelines-test.md @@ -1 +1 @@ -* Add the `databricks pipelines test` command, which runs a pipeline's tests in a test-only update. +* Add the `databricks pipelines test` command, which runs a pipeline's tests in a test-only update. ([#7271](https://github.com/databricks/cli/pull/7271)) From 683cedee5e84521c9049949695b276ebb7fbe766 Mon Sep 17 00:00:00 2001 From: Bhukya Date: Sat, 31 Oct 2026 15:52:32 +0000 Subject: [PATCH 06/10] Reject a pipelines test -k or -m value that starts with - --- cmd/pipelines/test.go | 4 ++++ cmd/pipelines/test_test.go | 3 ++- 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/cmd/pipelines/test.go b/cmd/pipelines/test.go index 373ebab8be4..b3118b32daf 100644 --- a/cmd/pipelines/test.go +++ b/cmd/pipelines/test.go @@ -51,6 +51,7 @@ func newStartTestUpdateRequest(selection, pytestArgs []string) startTestUpdateRe const pytestArgHint = "pass pytest arguments with --pytest-arg, for example --pytest-arg=-k --pytest-arg='orders and not slow'" // validatePytestArgs accepts only -k and -m, each with a value in the next argument or attached to it. +// A value in the next argument cannot start with "-", because pytest reads it as an option. func validatePytestArgs(args []string) error { for i := 0; i < len(args); i++ { arg := args[i] @@ -59,6 +60,9 @@ func validatePytestArgs(args []string) error { if i == len(args)-1 { return fmt.Errorf("pytest argument %s requires a value: pass it in the next --pytest-arg", arg) } + if next := args[i+1]; strings.HasPrefix(next, "-") { + return fmt.Errorf("pytest argument %s requires a value, not %q: pass it in the next --pytest-arg", arg, next) + } i++ case strings.HasPrefix(arg, "-k") || strings.HasPrefix(arg, "-m"): case !strings.HasPrefix(arg, "-"): diff --git a/cmd/pipelines/test_test.go b/cmd/pipelines/test_test.go index fc67a1b879f..8b951b2c46a 100644 --- a/cmd/pipelines/test_test.go +++ b/cmd/pipelines/test_test.go @@ -22,7 +22,8 @@ func TestValidatePytestArgs(t *testing.T) { {args: nil}, {args: []string{"-k", "orders and not slow", "-m", "smoke,fast"}}, {args: []string{"-korders", "-msmoke"}}, - {args: []string{"-k", "-m"}}, + {args: []string{"-k", "-m"}, wantErr: `pytest argument -k requires a value, not "-m": pass it in the next --pytest-arg`}, + {args: []string{"-m", "--maxfail=1"}, wantErr: `pytest argument -m requires a value, not "--maxfail=1": pass it in the next --pytest-arg`}, {args: []string{"-k"}, wantErr: "pytest argument -k requires a value: pass it in the next --pytest-arg"}, {args: []string{"-m", "smoke", "-m"}, wantErr: "pytest argument -m requires a value: pass it in the next --pytest-arg"}, {args: []string{"-x"}, wantErr: `unsupported pytest argument "-x": only -k and -m are supported`}, From 368ddfed7f626e06b7fd0e57b9130a96c2ff32fd Mon Sep 17 00:00:00 2001 From: Bhukya Date: Sat, 31 Oct 2026 16:00:47 +0000 Subject: [PATCH 07/10] Recommend pipelines deploy only after pipelines test finds the pipeline --- acceptance/pipelines/test/selection/output.txt | 4 ---- cmd/pipelines/test.go | 2 +- 2 files changed, 1 insertion(+), 5 deletions(-) diff --git a/acceptance/pipelines/test/selection/output.txt b/acceptance/pipelines/test/selection/output.txt index 46c3137cecc..6e188b5d08a 100644 --- a/acceptance/pipelines/test/selection/output.txt +++ b/acceptance/pipelines/test/selection/output.txt @@ -105,10 +105,6 @@ Update: [UUID] === Test selections require the pipeline KEY first >>> musterr [CLI] pipelines test tests/test_orders.py -Recommendation: This command runs the last deployed version of the code - -If you've made local changes, run 'databricks pipelines deploy' first to ensure they are included. - Error: resource with key "tests/test_orders.py" not found; the first argument must be a pipeline KEY when test selections are given === Test selections must be in the synchronized files diff --git a/cmd/pipelines/test.go b/cmd/pipelines/test.go index b3118b32daf..061e70c44ce 100644 --- a/cmd/pipelines/test.go +++ b/cmd/pipelines/test.go @@ -341,7 +341,6 @@ func resolveTestTargetInBundle(cmd *cobra.Command, args []string) (testTarget, e return testTarget{}, err } ctx := cmd.Context() - suggestPipelineDeploy(ctx, cmd) if _, err := resources.Lookup(b, key, isPipeline); err != nil { return testTarget{}, fmt.Errorf("%w; the first argument must be a pipeline KEY when test selections are given", err) @@ -350,6 +349,7 @@ func resolveTestTargetInBundle(cmd *cobra.Command, args []string) (testTarget, e if err != nil { return testTarget{}, err } + suggestPipelineDeploy(ctx, cmd) cwd, err := os.Getwd() if err != nil { From fd40c0ccf3eb267729e33b9e9e0df077c42c8df7 Mon Sep 17 00:00:00 2001 From: Bhukya Date: Sat, 31 Oct 2026 16:19:18 +0000 Subject: [PATCH 08/10] Resolve pipelines test selections as local paths or against root_path --- .../pipelines/test/arguments/output.txt | 10 +- .../pipelines/test/pipeline-id/output.txt | 2 +- .../pipelines/test/selection/output.txt | 8 + acceptance/pipelines/test/selection/script | 3 + .../pipelines/test/test-ids/databricks.yml | 11 -- .../pipelines/test/test-ids/out.test.toml | 1 - acceptance/pipelines/test/test-ids/output.txt | 46 ----- .../test/test-ids/pipeline/tests/test_a.py | 2 - .../pipeline/transformations/orders.py | 1 - acceptance/pipelines/test/test-ids/script | 16 -- cmd/pipelines/test.go | 166 +++++------------- cmd/pipelines/test_test.go | 101 +++-------- 12 files changed, 84 insertions(+), 283 deletions(-) delete mode 100644 acceptance/pipelines/test/test-ids/databricks.yml delete mode 100644 acceptance/pipelines/test/test-ids/out.test.toml delete mode 100644 acceptance/pipelines/test/test-ids/output.txt delete mode 100644 acceptance/pipelines/test/test-ids/pipeline/tests/test_a.py delete mode 100644 acceptance/pipelines/test/test-ids/pipeline/transformations/orders.py delete mode 100644 acceptance/pipelines/test/test-ids/script diff --git a/acceptance/pipelines/test/arguments/output.txt b/acceptance/pipelines/test/arguments/output.txt index 4c3eed3f7fd..953f88b1e31 100644 --- a/acceptance/pipelines/test/arguments/output.txt +++ b/acceptance/pipelines/test/arguments/output.txt @@ -9,11 +9,11 @@ Outside a project, identify the pipeline with --pipeline-id; all positional arguments are then test selections. TEST_SELECTION is a test directory, file, or pytest test ID, for example -tests/test_orders.py::test_backfill. A relative path that exists locally is -converted to its deployed workspace path. Any other relative path is a test ID -as the results show it: relative to the deepest folder that holds the -pipeline's root_path and its test sources. Paths that start with / are -workspace paths. Without a selection, all of the pipeline's tests run. +tests/test_orders.py::test_backfill. In a project, a relative path is a local +path, relative to the working directory, and is converted to its deployed +workspace path. With --pipeline-id, a relative path is relative to the +pipeline's root_path. Paths that start with / are workspace paths. Without a +selection, all of the pipeline's tests run. Each --pytest-arg passes one argument to pytest, in the order given. Only -k and -m are supported, for example: diff --git a/acceptance/pipelines/test/pipeline-id/output.txt b/acceptance/pipelines/test/pipeline-id/output.txt index 2952c195601..9138ee2e806 100644 --- a/acceptance/pipelines/test/pipeline-id/output.txt +++ b/acceptance/pipelines/test/pipeline-id/output.txt @@ -21,7 +21,7 @@ Update: [UUID] === Without a root_path, test selections must be workspace paths >>> musterr [CLI] pipelines test --pipeline-id [UUID] tests -Error: cannot resolve test selection "tests": the pipeline has no root_path or testing.sources; use an absolute workspace path such as /Workspace/Users/someone@example.com/tests +Error: cannot resolve test selection "tests": the pipeline has no root_path; use an absolute workspace path such as /Workspace/Users/someone@example.com/tests >>> [CLI] pipelines test --pipeline-id [UUID] /Workspace/Shared/tests/test_shared.py::test_x Pipeline: [UUID] diff --git a/acceptance/pipelines/test/selection/output.txt b/acceptance/pipelines/test/selection/output.txt index 6e188b5d08a..61f940aee57 100644 --- a/acceptance/pipelines/test/selection/output.txt +++ b/acceptance/pipelines/test/selection/output.txt @@ -114,3 +114,11 @@ Recommendation: This command runs the last deployed version of the code If you've made local changes, run 'databricks pipelines deploy' first to ensure they are included. Error: cannot resolve test selection "../test_outside.py": [TEST_TMP_DIR]_PARENT/test_outside.py is outside the synchronized bundle files in [TEST_TMP_DIR] + +=== Relative test selections must exist locally +>>> musterr [CLI] pipelines test my_pipeline tests/test_ordrs.py::test_backfill +Recommendation: This command runs the last deployed version of the code + +If you've made local changes, run 'databricks pipelines deploy' first to ensure they are included. + +Error: cannot resolve test selection "tests/test_ordrs.py::test_backfill": [TEST_TMP_DIR]/tests/test_ordrs.py does not exist; pass a local path or an absolute workspace path diff --git a/acceptance/pipelines/test/selection/script b/acceptance/pipelines/test/selection/script index ca96fb91835..2fbe58cb4ed 100644 --- a/acceptance/pipelines/test/selection/script +++ b/acceptance/pipelines/test/selection/script @@ -24,3 +24,6 @@ trace musterr $CLI pipelines test tests/test_orders.py title "Test selections must be in the synchronized files" trace musterr $CLI pipelines test my_pipeline ../test_outside.py + +title "Relative test selections must exist locally" +trace musterr $CLI pipelines test my_pipeline tests/test_ordrs.py::test_backfill diff --git a/acceptance/pipelines/test/test-ids/databricks.yml b/acceptance/pipelines/test/test-ids/databricks.yml deleted file mode 100644 index 423404790c4..00000000000 --- a/acceptance/pipelines/test/test-ids/databricks.yml +++ /dev/null @@ -1,11 +0,0 @@ -bundle: - name: test-pipeline-test-ids - -resources: - pipelines: - my_pipeline: - name: test-pipeline-test-ids - root_path: ./pipeline - libraries: - - file: - path: ./pipeline/transformations/orders.py diff --git a/acceptance/pipelines/test/test-ids/out.test.toml b/acceptance/pipelines/test/test-ids/out.test.toml deleted file mode 100644 index 18b1a88417e..00000000000 --- a/acceptance/pipelines/test/test-ids/out.test.toml +++ /dev/null @@ -1 +0,0 @@ -Cloud = false diff --git a/acceptance/pipelines/test/test-ids/output.txt b/acceptance/pipelines/test/test-ids/output.txt deleted file mode 100644 index f257359806e..00000000000 --- a/acceptance/pipelines/test/test-ids/output.txt +++ /dev/null @@ -1,46 +0,0 @@ - ->>> [CLI] pipelines deploy -Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-pipeline-test-ids/default/files... -Created pipelines.my_pipeline -Files: 3 uploaded, 0 deleted -Resources: 1 created, 0 changed, 0 deleted, 0 unchanged -View your pipeline my_pipeline here: [DATABRICKS_URL]/pipelines/[UUID]?w=[WORKSPACE_ID] - -=== A test ID resolves against the folder that test IDs are relative to - ->>> [CLI] pipelines test my_pipeline tests/test_a.py::test_one -Recommendation: This command runs the last deployed version of the code - -If you've made local changes, run 'databricks pipelines deploy' first to ensure they are included. - -Pipeline: my_pipeline ([UUID]) -Update: [UUID] - ->>> print_requests.py //updates -{"test_selection":["/Workspace/Users/[USERNAME]/.bundle/test-pipeline-test-ids/default/files/pipeline/tests/test_a.py::test_one"]} - -=== The same ID from the pipeline folder, where it is a local path - ->>> [CLI] pipelines test my_pipeline tests/test_a.py::test_one -Recommendation: This command runs the last deployed version of the code - -If you've made local changes, run 'databricks pipelines deploy' first to ensure they are included. - -Pipeline: my_pipeline ([UUID]) -Update: [UUID] - ->>> print_requests.py //updates -{"test_selection":["/Workspace/Users/[USERNAME]/.bundle/test-pipeline-test-ids/default/files/pipeline/tests/test_a.py::test_one"]} - -=== A local path from the bundle root - ->>> [CLI] pipelines test my_pipeline pipeline/tests/test_a.py -Recommendation: This command runs the last deployed version of the code - -If you've made local changes, run 'databricks pipelines deploy' first to ensure they are included. - -Pipeline: my_pipeline ([UUID]) -Update: [UUID] - ->>> print_requests.py //updates -{"test_selection":["/Workspace/Users/[USERNAME]/.bundle/test-pipeline-test-ids/default/files/pipeline/tests/test_a.py"]} diff --git a/acceptance/pipelines/test/test-ids/pipeline/tests/test_a.py b/acceptance/pipelines/test/test-ids/pipeline/tests/test_a.py deleted file mode 100644 index 65de24b9c57..00000000000 --- a/acceptance/pipelines/test/test-ids/pipeline/tests/test_a.py +++ /dev/null @@ -1,2 +0,0 @@ -def test_one(): - pass diff --git a/acceptance/pipelines/test/test-ids/pipeline/transformations/orders.py b/acceptance/pipelines/test/test-ids/pipeline/transformations/orders.py deleted file mode 100644 index 7d4290a117a..00000000000 --- a/acceptance/pipelines/test/test-ids/pipeline/transformations/orders.py +++ /dev/null @@ -1 +0,0 @@ -x = 1 diff --git a/acceptance/pipelines/test/test-ids/script b/acceptance/pipelines/test/test-ids/script deleted file mode 100644 index 3972d69e926..00000000000 --- a/acceptance/pipelines/test/test-ids/script +++ /dev/null @@ -1,16 +0,0 @@ -trace $CLI pipelines deploy -rm "$OUT_REQUESTS" - -title "A test ID resolves against the folder that test IDs are relative to\n" -trace $CLI pipelines test my_pipeline tests/test_a.py::test_one -trace print_requests.py //updates | jq -c .body.test_options - -title "The same ID from the pipeline folder, where it is a local path\n" -cd pipeline -trace $CLI pipelines test my_pipeline tests/test_a.py::test_one -cd .. -trace print_requests.py //updates | jq -c .body.test_options - -title "A local path from the bundle root\n" -trace $CLI pipelines test my_pipeline pipeline/tests/test_a.py -trace print_requests.py //updates | jq -c .body.test_options diff --git a/cmd/pipelines/test.go b/cmd/pipelines/test.go index 061e70c44ce..32542c459e6 100644 --- a/cmd/pipelines/test.go +++ b/cmd/pipelines/test.go @@ -5,6 +5,7 @@ import ( "errors" "fmt" "io" + "io/fs" "net/http" "os" "path" @@ -136,24 +137,29 @@ func bundleFilesRoot(b *bundle.Bundle) string { } } -// bundleTestSelectionPath maps a relative selection to a workspace path. A path that exists locally, -// relative to the working directory, maps to its deployed path under filesRoot. Otherwise the -// selection is a test ID that the CLI printed, relative to testIDBase, unless it leaves the working -// directory. -func bundleTestSelectionPath(cwd, syncRoot, filesRoot, testIDBase, localPath string) (string, error) { - rel := path.Clean(filepath.ToSlash(localPath)) - leavesCwd := rel == ".." || strings.HasPrefix(rel, "../") - if _, err := os.Stat(filepath.Join(cwd, localPath)); err != nil && !leavesCwd && testIDBase != "" { - return testIDBasePath(testIDBase, localPath) - } +// bundleTestSelectionPath maps a local path, relative to the working directory, to its deployed path +// under filesRoot. The path must exist locally, so that a typo fails before compute starts. +func bundleTestSelectionPath(cwd, syncRoot, filesRoot, localPath string) (string, error) { if filesRoot == "" { - return "", errors.New("local test paths are not supported with workspace.sync_mode immutable; pass the test ID that the results show or an absolute workspace path") + return "", errors.New("local test paths are not supported with workspace.sync_mode immutable; pass an absolute workspace path") + } + workspacePath, err := bundleWorkspacePath(cwd, syncRoot, filesRoot, localPath) + if err != nil { + return "", err + } + if !filepath.IsAbs(localPath) { + localPath = filepath.Join(cwd, localPath) + } + if _, err := os.Stat(localPath); errors.Is(err, fs.ErrNotExist) { + return "", fmt.Errorf("%s does not exist; pass a local path or an absolute workspace path", filepath.ToSlash(localPath)) + } else if err != nil { + return "", err } - return bundleWorkspacePath(cwd, syncRoot, filesRoot, localPath) + return workspacePath, nil } -// needsTestIDBase reports whether a selection is a relative path, which may be a test ID. -func needsTestIDBase(selections []string) bool { +// hasRelativeSelection reports whether a selection is a relative path. +func hasRelativeSelection(selections []string) bool { for _, selection := range selections { if p, _, _ := strings.Cut(selection, "::"); !strings.HasPrefix(p, "/") { return true @@ -162,95 +168,24 @@ func needsTestIDBase(selections []string) bool { return false } -// testIDBase returns the folder that the service names test cases relative to, or "" if the -// pipeline has neither a root_path nor test sources. pytest runs with this folder as its rootdir: -// the deepest folder that holds the root path and every test source or, without a root path, the -// folder of every test source. -func testIDBase(rootPath string, sources []string) string { - var folders [][]string - if rootPath != "" { - folders = append(folders, pathSegments(normalizeWorkspacePath(rootPath))) +// rootPathSelection resolves a relative path against the pipeline's root_path. +func rootPathSelection(rootPath, relPath string) (string, error) { + if rootPath == "" { + return "", errors.New("the pipeline has no root_path; use an absolute workspace path such as /Workspace/Users/someone@example.com/tests") } - for _, source := range sources { - segments := pathSegments(normalizeWorkspacePath(strings.TrimSuffix(source, "/**"))) - if rootPath == "" && len(segments) > 0 { - segments = segments[:len(segments)-1] - } - folders = append(folders, segments) - } - if len(folders) == 0 { - return "" - } - common := folders[0] - for _, folder := range folders[1:] { - n := 0 - for n < len(common) && n < len(folder) && common[n] == folder[n] { - n++ - } - common = common[:n] - } - return "/" + strings.Join(common, "/") -} - -// normalizeWorkspacePath adds the /Workspace prefix to an absolute path without it, as the service does. -func normalizeWorkspacePath(p string) string { - if strings.HasPrefix(p, "/") && !strings.HasPrefix(p, "/Workspace") { - return "/Workspace" + p - } - return p + return path.Join(rootPath, filepath.ToSlash(relPath)), nil } -func pathSegments(p string) []string { - var segments []string - for segment := range strings.SplitSeq(p, "/") { - if segment != "" { - segments = append(segments, segment) - } - } - return segments -} - -// testIDBasePath resolves a relative path against the folder that test IDs are relative to. -func testIDBasePath(base, relPath string) (string, error) { - if base == "" { - return "", errors.New("the pipeline has no root_path or testing.sources; use an absolute workspace path such as /Workspace/Users/someone@example.com/tests") - } - return path.Join(base, filepath.ToSlash(relPath)), nil -} - -// pipelineTestPaths holds the fields of a Get Pipeline response that decide the folder of test IDs. -// The Go SDK has no spec.testing yet, so the response is decoded from the raw API call. -type pipelineTestPaths struct { - Spec struct { - RootPath string `json:"root_path"` - Testing struct { - Sources []struct { - Glob struct { - Include string `json:"include"` - } `json:"glob"` - } `json:"sources"` - } `json:"testing"` - } `json:"spec"` -} - -// getTestIDBase reads the deployed pipeline and returns the folder that its test IDs are relative to. -func getTestIDBase(ctx context.Context, w *databricks.WorkspaceClient, pipelineID string) (string, error) { - apiClient, err := client.New(w.Config) - if err != nil { - return "", fmt.Errorf("failed to create API client: %w", err) - } - var response pipelineTestPaths - err = apiClient.Do(ctx, http.MethodGet, "/api/2.0/pipelines/"+pipelineID, auth.WorkspaceIDHeaders(w.Config), nil, nil, &response) +// getRootPath reads the root_path of the deployed pipeline. +func getRootPath(ctx context.Context, w *databricks.WorkspaceClient, pipelineID string) (string, error) { + pipeline, err := w.Pipelines.Get(ctx, pipelines.GetPipelineRequest{PipelineId: pipelineID}) if err != nil { return "", err } - var sources []string - for _, source := range response.Spec.Testing.Sources { - if source.Glob.Include != "" { - sources = append(sources, source.Glob.Include) - } + if pipeline.Spec == nil { + return "", nil } - return testIDBase(response.Spec.RootPath, sources), nil + return pipeline.Spec.RootPath, nil } // startTestUpdate starts a test-only update and returns its ID. @@ -293,14 +228,6 @@ func (t testTarget) printHeader(out io.Writer, updateID string) { fmt.Fprintf(out, "Update: %s\n", updateID) } -// readTestIDBase reads the folder of test IDs from the deployed pipeline, if a selection needs it. -func readTestIDBase(ctx context.Context, w *databricks.WorkspaceClient, pipelineID string, selections []string) (string, error) { - if !needsTestIDBase(selections) { - return "", nil - } - return getTestIDBase(ctx, w, pipelineID) -} - // resolveTestTargetByID identifies the pipeline by --pipeline-id; all positional arguments are selections. func resolveTestTargetByID(cmd *cobra.Command, args []string, pipelineID string) (testTarget, error) { if cmd.Flag("target").Value.String() == "" { @@ -312,12 +239,16 @@ func resolveTestTargetByID(cmd *cobra.Command, args []string, pipelineID string) ctx := cmd.Context() w := cmdctx.WorkspaceClient(ctx) - base, err := readTestIDBase(ctx, w, pipelineID, args) - if err != nil { - return testTarget{}, err + var rootPath string + if hasRelativeSelection(args) { + var err error + rootPath, err = getRootPath(ctx, w, pipelineID) + if err != nil { + return testTarget{}, err + } } selections, err := resolveTestSelections(args, func(p string) (string, error) { - return testIDBasePath(base, p) + return rootPathSelection(rootPath, p) }) if err != nil { return testTarget{}, err @@ -359,19 +290,14 @@ func resolveTestTargetInBundle(cmd *cobra.Command, args []string) (testTarget, e if len(args) > 1 { selectionArgs = args[1:] } - w := b.WorkspaceClient(ctx) - base, err := readTestIDBase(ctx, w, pipelineID, selectionArgs) - if err != nil { - return testTarget{}, err - } filesRoot := bundleFilesRoot(b) selections, err := resolveTestSelections(selectionArgs, func(p string) (string, error) { - return bundleTestSelectionPath(cwd, b.SyncRootPath, filesRoot, base, p) + return bundleTestSelectionPath(cwd, b.SyncRootPath, filesRoot, p) }) if err != nil { return testTarget{}, err } - return testTarget{key: key, pipelineID: pipelineID, selections: selections, w: w}, nil + return testTarget{key: key, pipelineID: pipelineID, selections: selections, w: b.WorkspaceClient(ctx)}, nil } func testCommand() *cobra.Command { @@ -387,11 +313,11 @@ Outside a project, identify the pipeline with --pipeline-id; all positional arguments are then test selections. TEST_SELECTION is a test directory, file, or pytest test ID, for example -tests/test_orders.py::test_backfill. A relative path that exists locally is -converted to its deployed workspace path. Any other relative path is a test ID -as the results show it: relative to the deepest folder that holds the -pipeline's root_path and its test sources. Paths that start with / are -workspace paths. Without a selection, all of the pipeline's tests run. +tests/test_orders.py::test_backfill. In a project, a relative path is a local +path, relative to the working directory, and is converted to its deployed +workspace path. With --pipeline-id, a relative path is relative to the +pipeline's root_path. Paths that start with / are workspace paths. Without a +selection, all of the pipeline's tests run. Each --pytest-arg passes one argument to pytest, in the order given. Only -k and -m are supported, for example: diff --git a/cmd/pipelines/test_test.go b/cmd/pipelines/test_test.go index 8b951b2c46a..ffc845ca551 100644 --- a/cmd/pipelines/test_test.go +++ b/cmd/pipelines/test_test.go @@ -96,64 +96,13 @@ func TestBundleWorkspacePath(t *testing.T) { assert.ErrorContains(t, err, "is outside the synchronized bundle files in "+syncRoot) } -func TestTestIDBase(t *testing.T) { - tests := []struct { - name string - rootPath string - sources []string - want string - }{ - { - name: "deepest folder holding the root path and every test source", - rootPath: "/Workspace/Users/u/files/src/my_pipeline", - sources: []string{"/Workspace/Users/u/files/tests/**", "/Workspace/Users/u/files/checks/test_x.py"}, - want: "/Workspace/Users/u/files", - }, - { - name: "root path that holds the test sources", - rootPath: "/Users/u/sl", - sources: []string{"/Users/u/sl/tests/**"}, - want: "/Workspace/Users/u/sl", - }, - { - name: "without a root path, the folder of a file test source", - sources: []string{"/Workspace/Users/u/tests/test_x.py"}, - want: "/Workspace/Users/u/tests", - }, - { - name: "without a root path, the deepest folder holding the folders of every test source", - sources: []string{"/Workspace/Users/u/a/tests/**", "/Workspace/Users/u/b/tests/**"}, - want: "/Workspace/Users/u", - }, - { - name: "without a root path, the parent of a folder test source", - sources: []string{"/Workspace/Users/u/tests/**"}, - want: "/Workspace/Users/u", - }, - { - name: "root path without test sources", - rootPath: "/Workspace/Users/u/orders", - want: "/Workspace/Users/u/orders", - }, - { - name: "neither", - want: "", - }, - } - for _, tc := range tests { - t.Run(tc.name, func(t *testing.T) { - assert.Equal(t, tc.want, testIDBase(tc.rootPath, tc.sources)) - }) - } -} - -func TestTestIDBasePath(t *testing.T) { - got, err := testIDBasePath("/Workspace/Users/someone/orders", "tests/test_orders.py") +func TestRootPathSelection(t *testing.T) { + got, err := rootPathSelection("/Workspace/Users/someone/orders", "tests/test_orders.py") require.NoError(t, err) assert.Equal(t, "/Workspace/Users/someone/orders/tests/test_orders.py", got) - _, err = testIDBasePath("", "tests") - assert.EqualError(t, err, "the pipeline has no root_path or testing.sources; use an absolute workspace path such as /Workspace/Users/someone@example.com/tests") + _, err = rootPathSelection("", "tests") + assert.EqualError(t, err, "the pipeline has no root_path; use an absolute workspace path such as /Workspace/Users/someone@example.com/tests") } func TestBundleFilesRoot(t *testing.T) { @@ -181,41 +130,33 @@ func TestBundleTestSelectionPath(t *testing.T) { require.NoError(t, os.MkdirAll(filepath.Join(syncRoot, "pipeline", "tests"), 0o755)) require.NoError(t, os.WriteFile(filepath.Join(syncRoot, "pipeline", "tests", "test_a.py"), nil, 0o644)) filesRoot := "/Workspace/Users/someone/.bundle/orders/dev/files" - base := filesRoot + "/pipeline" - // A path that exists where the user stands is a local path. - got, err := bundleTestSelectionPath(filepath.Join(syncRoot, "pipeline"), syncRoot, filesRoot, base, "tests/test_a.py") + // A local path maps to its deployed path, relative to where the user stands. + got, err := bundleTestSelectionPath(filepath.Join(syncRoot, "pipeline"), syncRoot, filesRoot, "tests/test_a.py") require.NoError(t, err) assert.Equal(t, filesRoot+"/pipeline/tests/test_a.py", got) - // Otherwise it is a test ID, relative to the folder that test IDs are relative to. - got, err = bundleTestSelectionPath(syncRoot, syncRoot, filesRoot, base, "tests/test_a.py") + got, err = bundleTestSelectionPath(syncRoot, syncRoot, filesRoot, "pipeline/tests") require.NoError(t, err) - assert.Equal(t, filesRoot+"/pipeline/tests/test_a.py", got) + assert.Equal(t, filesRoot+"/pipeline/tests", got) - // Without that folder, it is a local path. - got, err = bundleTestSelectionPath(syncRoot, syncRoot, filesRoot, "", "tests/test_a.py") - require.NoError(t, err) - assert.Equal(t, filesRoot+"/tests/test_a.py", got) + // A path that does not exist locally is an error, so that a typo fails before compute starts. + _, err = bundleTestSelectionPath(syncRoot, syncRoot, filesRoot, "tests/test_a.py") + assert.EqualError(t, err, filepath.ToSlash(filepath.Join(syncRoot, "tests", "test_a.py"))+" does not exist; pass a local path or an absolute workspace path") - // A path that leaves that folder is a local path too. The error shows forward slashes. - for _, testIDBase := range []string{base, ""} { - _, err = bundleTestSelectionPath(syncRoot, syncRoot, filesRoot, testIDBase, filepath.Join("..", "tests")) - assert.ErrorContains(t, err, "is outside the synchronized bundle files in "+filepath.ToSlash(syncRoot)) - } + // A path outside the synchronized files is an error. The error shows forward slashes. + _, err = bundleTestSelectionPath(syncRoot, syncRoot, filesRoot, filepath.Join("..", "tests")) + assert.ErrorContains(t, err, "is outside the synchronized bundle files in "+filepath.ToSlash(syncRoot)) - // When the deployed path of local files is unknown, a local path is an error, but a test ID still works. - _, err = bundleTestSelectionPath(filepath.Join(syncRoot, "pipeline"), syncRoot, "", base, "tests/test_a.py") - assert.EqualError(t, err, "local test paths are not supported with workspace.sync_mode immutable; pass the test ID that the results show or an absolute workspace path") - got, err = bundleTestSelectionPath(syncRoot, syncRoot, "", base, "tests/test_a.py") - require.NoError(t, err) - assert.Equal(t, filesRoot+"/pipeline/tests/test_a.py", got) + // When the deployed path of local files is unknown, a local path is an error. + _, err = bundleTestSelectionPath(filepath.Join(syncRoot, "pipeline"), syncRoot, "", "tests/test_a.py") + assert.EqualError(t, err, "local test paths are not supported with workspace.sync_mode immutable; pass an absolute workspace path") } -func TestNeedsTestIDBase(t *testing.T) { - assert.False(t, needsTestIDBase(nil)) - assert.False(t, needsTestIDBase([]string{"/Workspace/tests", "/Workspace/a.py::test_x"})) - assert.True(t, needsTestIDBase([]string{"/Workspace/tests", "tests/test_a.py::test_x"})) +func TestHasRelativeSelection(t *testing.T) { + assert.False(t, hasRelativeSelection(nil)) + assert.False(t, hasRelativeSelection([]string{"/Workspace/tests", "/Workspace/a.py::test_x"})) + assert.True(t, hasRelativeSelection([]string{"/Workspace/tests", "tests/test_a.py::test_x"})) } func TestNewStartTestUpdateRequest(t *testing.T) { From b020fc599504b477580f24ebfbdaa3059224c60b Mon Sep 17 00:00:00 2001 From: Bhukya Date: Sat, 31 Oct 2026 16:37:46 +0000 Subject: [PATCH 09/10] Cover pipelines test selections with workspace.sync_mode immutable --- .../pipelines/test/immutable/databricks.yml | 13 +++++ .../pipelines/test/immutable/out.test.toml | 1 + .../pipelines/test/immutable/output.txt | 53 +++++++++++++++++++ .../pipelines/test/immutable/pipeline_file.py | 6 +++ acceptance/pipelines/test/immutable/script | 10 ++++ .../test/immutable/tests/test_orders.py | 2 + 6 files changed, 85 insertions(+) create mode 100644 acceptance/pipelines/test/immutable/databricks.yml create mode 100644 acceptance/pipelines/test/immutable/out.test.toml create mode 100644 acceptance/pipelines/test/immutable/output.txt create mode 100644 acceptance/pipelines/test/immutable/pipeline_file.py create mode 100644 acceptance/pipelines/test/immutable/script create mode 100644 acceptance/pipelines/test/immutable/tests/test_orders.py diff --git a/acceptance/pipelines/test/immutable/databricks.yml b/acceptance/pipelines/test/immutable/databricks.yml new file mode 100644 index 00000000000..483234d0dd4 --- /dev/null +++ b/acceptance/pipelines/test/immutable/databricks.yml @@ -0,0 +1,13 @@ +bundle: + name: test-pipeline-test-immutable + +workspace: + sync_mode: immutable + +resources: + pipelines: + my_pipeline: + name: test-pipeline-test-immutable + libraries: + - file: + path: pipeline_file.py diff --git a/acceptance/pipelines/test/immutable/out.test.toml b/acceptance/pipelines/test/immutable/out.test.toml new file mode 100644 index 00000000000..18b1a88417e --- /dev/null +++ b/acceptance/pipelines/test/immutable/out.test.toml @@ -0,0 +1 @@ +Cloud = false diff --git a/acceptance/pipelines/test/immutable/output.txt b/acceptance/pipelines/test/immutable/output.txt new file mode 100644 index 00000000000..a162096897f --- /dev/null +++ b/acceptance/pipelines/test/immutable/output.txt @@ -0,0 +1,53 @@ + +>>> [CLI] pipelines deploy +Created internal_immutable_snapshots.immutable +Created pipelines.my_pipeline +Files: 0 uploaded, 0 deleted +Resources: 2 created, 0 changed, 0 deleted, 0 unchanged +View your pipeline my_pipeline here: [DATABRICKS_URL]/pipelines/[UUID]?w=[WORKSPACE_ID] + +=== Local test paths are not supported with workspace.sync_mode immutable +>>> musterr [CLI] pipelines test my_pipeline tests/test_orders.py::test_backfill +Recommendation: This command runs the last deployed version of the code + +If you've made local changes, run 'databricks pipelines deploy' first to ensure they are included. + +Error: cannot resolve test selection "tests/test_orders.py::test_backfill": local test paths are not supported with workspace.sync_mode immutable; pass an absolute workspace path + +=== Absolute workspace paths and running all tests still work +>>> [CLI] pipelines test my_pipeline /Workspace/Users/someone@example.com/tests/test_orders.py::test_backfill +Recommendation: This command runs the last deployed version of the code + +If you've made local changes, run 'databricks pipelines deploy' first to ensure they are included. + +Pipeline: my_pipeline ([UUID]) +Update: [UUID] + +>>> [CLI] pipelines test my_pipeline +Recommendation: This command runs the last deployed version of the code + +If you've made local changes, run 'databricks pipelines deploy' first to ensure they are included. + +Pipeline: my_pipeline ([UUID]) +Update: [UUID] + +>>> print_requests.py //updates +{ + "method": "POST", + "path": "/api/2.0/pipelines/[UUID]/updates", + "body": { + "test_only": true, + "test_options": { + "test_selection": [ + "/Workspace/Users/someone@example.com/tests/test_orders.py::test_backfill" + ] + } + } +} +{ + "method": "POST", + "path": "/api/2.0/pipelines/[UUID]/updates", + "body": { + "test_only": true + } +} diff --git a/acceptance/pipelines/test/immutable/pipeline_file.py b/acceptance/pipelines/test/immutable/pipeline_file.py new file mode 100644 index 00000000000..4a7a3a1878e --- /dev/null +++ b/acceptance/pipelines/test/immutable/pipeline_file.py @@ -0,0 +1,6 @@ +from pyspark import pipelines as dp + + +@dp.table +def orders(): + return spark.range(10) diff --git a/acceptance/pipelines/test/immutable/script b/acceptance/pipelines/test/immutable/script new file mode 100644 index 00000000000..972ac6e3e45 --- /dev/null +++ b/acceptance/pipelines/test/immutable/script @@ -0,0 +1,10 @@ +trace $CLI pipelines deploy +rm "$OUT_REQUESTS" + +title "Local test paths are not supported with workspace.sync_mode immutable" +trace musterr $CLI pipelines test my_pipeline tests/test_orders.py::test_backfill + +title "Absolute workspace paths and running all tests still work" +trace $CLI pipelines test my_pipeline /Workspace/Users/someone@example.com/tests/test_orders.py::test_backfill +trace $CLI pipelines test my_pipeline +trace print_requests.py //updates diff --git a/acceptance/pipelines/test/immutable/tests/test_orders.py b/acceptance/pipelines/test/immutable/tests/test_orders.py new file mode 100644 index 00000000000..ec31eaa1b01 --- /dev/null +++ b/acceptance/pipelines/test/immutable/tests/test_orders.py @@ -0,0 +1,2 @@ +def test_backfill(): + assert True From 8a916d9a3776b242c50ac3173de743870ae1e85e Mon Sep 17 00:00:00 2001 From: Bhukya Date: Sat, 31 Oct 2026 17:14:52 +0000 Subject: [PATCH 10/10] Require pipelines test -k and -m values in the next --pytest-arg, as the API does --- acceptance/pipelines/test/arguments/output.txt | 10 +++++++++- acceptance/pipelines/test/arguments/script | 4 ++++ acceptance/pipelines/test/selection/output.txt | 5 +++-- acceptance/pipelines/test/selection/script | 2 +- cmd/pipelines/test.go | 16 ++++++++++++---- cmd/pipelines/test_test.go | 5 ++++- 6 files changed, 33 insertions(+), 9 deletions(-) diff --git a/acceptance/pipelines/test/arguments/output.txt b/acceptance/pipelines/test/arguments/output.txt index 953f88b1e31..73804fcebfc 100644 --- a/acceptance/pipelines/test/arguments/output.txt +++ b/acceptance/pipelines/test/arguments/output.txt @@ -16,7 +16,8 @@ pipeline's root_path. Paths that start with / are workspace paths. Without a selection, all of the pipeline's tests run. Each --pytest-arg passes one argument to pytest, in the order given. Only -k -and -m are supported, for example: +and -m are supported, each followed by its value in the next --pytest-arg, for +example: databricks pipelines test my_pipeline --pytest-arg=-k --pytest-arg='orders and not slow' @@ -57,6 +58,13 @@ Error: pytest argument -k requires a value: pass it in the next --pytest-arg >>> musterr [CLI] pipelines test --pipeline-id 1234 --pytest-arg=tests/test_orders.py Error: unsupported pytest argument "tests/test_orders.py": pass test paths and test IDs as TEST_SELECTION arguments +=== -k and -m take their value in the next --pytest-arg +>>> musterr [CLI] pipelines test --pipeline-id 1234 --pytest-arg=-korders +Error: pytest argument "-korders" must be split: pass --pytest-arg=-k --pytest-arg=orders + +>>> musterr [CLI] pipelines test --pipeline-id 1234 --pytest-arg=-k --pytest-arg=-m +Error: pytest argument -k requires a value, not "-m": pass it in the next --pytest-arg + === JSON output is not supported >>> musterr [CLI] pipelines test --pipeline-id 1234 -o json Error: --output json is not supported by pipelines test diff --git a/acceptance/pipelines/test/arguments/script b/acceptance/pipelines/test/arguments/script index dfe93d5c4b6..c4fd234f51c 100644 --- a/acceptance/pipelines/test/arguments/script +++ b/acceptance/pipelines/test/arguments/script @@ -10,5 +10,9 @@ trace musterr $CLI pipelines test --pipeline-id 1234 --pytest-arg=-x trace musterr $CLI pipelines test --pipeline-id 1234 --pytest-arg=-k trace musterr $CLI pipelines test --pipeline-id 1234 --pytest-arg=tests/test_orders.py +title "-k and -m take their value in the next --pytest-arg" +trace musterr $CLI pipelines test --pipeline-id 1234 --pytest-arg=-korders +trace musterr $CLI pipelines test --pipeline-id 1234 --pytest-arg=-k --pytest-arg=-m + title "JSON output is not supported" trace musterr $CLI pipelines test --pipeline-id 1234 -o json diff --git a/acceptance/pipelines/test/selection/output.txt b/acceptance/pipelines/test/selection/output.txt index 61f940aee57..ea657be82b5 100644 --- a/acceptance/pipelines/test/selection/output.txt +++ b/acceptance/pipelines/test/selection/output.txt @@ -25,7 +25,7 @@ Update: [UUID] } === Select tests by relative path and test ID, with pytest arguments ->>> [CLI] pipelines test my_pipeline tests tests/test_orders.py::test_backfill /Workspace/Shared/tests/test_shared.py --pytest-arg=-k --pytest-arg=orders and not slow --pytest-arg=-msmoke,fast +>>> [CLI] pipelines test my_pipeline tests tests/test_orders.py::test_backfill /Workspace/Shared/tests/test_shared.py --pytest-arg=-k --pytest-arg=orders and not slow --pytest-arg=-m --pytest-arg=smoke,fast Recommendation: This command runs the last deployed version of the code If you've made local changes, run 'databricks pipelines deploy' first to ensure they are included. @@ -48,7 +48,8 @@ Update: [UUID] "pytest_args": [ "-k", "orders and not slow", - "-msmoke,fast" + "-m", + "smoke,fast" ] } } diff --git a/acceptance/pipelines/test/selection/script b/acceptance/pipelines/test/selection/script index 2fbe58cb4ed..264927f6685 100644 --- a/acceptance/pipelines/test/selection/script +++ b/acceptance/pipelines/test/selection/script @@ -6,7 +6,7 @@ trace $CLI pipelines test trace print_requests.py //updates title "Select tests by relative path and test ID, with pytest arguments" -trace $CLI pipelines test my_pipeline tests tests/test_orders.py::test_backfill /Workspace/Shared/tests/test_shared.py --pytest-arg=-k --pytest-arg='orders and not slow' --pytest-arg=-msmoke,fast +trace $CLI pipelines test my_pipeline tests tests/test_orders.py::test_backfill /Workspace/Shared/tests/test_shared.py --pytest-arg=-k --pytest-arg='orders and not slow' --pytest-arg=-m --pytest-arg=smoke,fast trace print_requests.py //updates title "--pytest-arg also takes its value as the next argument" diff --git a/cmd/pipelines/test.go b/cmd/pipelines/test.go index 32542c459e6..0e75a73c66a 100644 --- a/cmd/pipelines/test.go +++ b/cmd/pipelines/test.go @@ -22,6 +22,7 @@ import ( "github.com/databricks/cli/libs/cmdgroup" "github.com/databricks/cli/libs/flags" "github.com/databricks/cli/libs/logdiag" + "github.com/databricks/cli/libs/shellquote" databricks "github.com/databricks/databricks-sdk-go" "github.com/databricks/databricks-sdk-go/client" "github.com/databricks/databricks-sdk-go/service/pipelines" @@ -51,8 +52,8 @@ func newStartTestUpdateRequest(selection, pytestArgs []string) startTestUpdateRe const pytestArgHint = "pass pytest arguments with --pytest-arg, for example --pytest-arg=-k --pytest-arg='orders and not slow'" -// validatePytestArgs accepts only -k and -m, each with a value in the next argument or attached to it. -// A value in the next argument cannot start with "-", because pytest reads it as an option. +// validatePytestArgs accepts only -k and -m, each followed by its value in the next argument, as the API +// requires. The value cannot be empty or start with "-", because pytest reads it as an option. func validatePytestArgs(args []string) error { for i := 0; i < len(args); i++ { arg := args[i] @@ -61,11 +62,17 @@ func validatePytestArgs(args []string) error { if i == len(args)-1 { return fmt.Errorf("pytest argument %s requires a value: pass it in the next --pytest-arg", arg) } - if next := args[i+1]; strings.HasPrefix(next, "-") { + next := args[i+1] + if strings.TrimSpace(next) == "" { + return fmt.Errorf("pytest argument %s requires a non-empty value", arg) + } + if strings.HasPrefix(next, "-") { return fmt.Errorf("pytest argument %s requires a value, not %q: pass it in the next --pytest-arg", arg, next) } i++ case strings.HasPrefix(arg, "-k") || strings.HasPrefix(arg, "-m"): + option, value := arg[:2], strings.TrimPrefix(arg[2:], "=") + return fmt.Errorf("pytest argument %q must be split: pass --pytest-arg=%s --pytest-arg=%s", arg, option, shellquote.BashArg(value)) case !strings.HasPrefix(arg, "-"): return fmt.Errorf("unsupported pytest argument %q: pass test paths and test IDs as TEST_SELECTION arguments", arg) default: @@ -320,7 +327,8 @@ pipeline's root_path. Paths that start with / are workspace paths. Without a selection, all of the pipeline's tests run. Each --pytest-arg passes one argument to pytest, in the order given. Only -k -and -m are supported, for example: +and -m are supported, each followed by its value in the next --pytest-arg, for +example: databricks pipelines test my_pipeline --pytest-arg=-k --pytest-arg='orders and not slow'`, } diff --git a/cmd/pipelines/test_test.go b/cmd/pipelines/test_test.go index ffc845ca551..a6ee609623a 100644 --- a/cmd/pipelines/test_test.go +++ b/cmd/pipelines/test_test.go @@ -21,7 +21,10 @@ func TestValidatePytestArgs(t *testing.T) { }{ {args: nil}, {args: []string{"-k", "orders and not slow", "-m", "smoke,fast"}}, - {args: []string{"-korders", "-msmoke"}}, + {args: []string{"-korders"}, wantErr: `pytest argument "-korders" must be split: pass --pytest-arg=-k --pytest-arg=orders`}, + {args: []string{"-msmoke,fast"}, wantErr: `pytest argument "-msmoke,fast" must be split: pass --pytest-arg=-m --pytest-arg='smoke,fast'`}, + {args: []string{"-k=orders and not slow"}, wantErr: `pytest argument "-k=orders and not slow" must be split: pass --pytest-arg=-k --pytest-arg='orders and not slow'`}, + {args: []string{"-k", " "}, wantErr: "pytest argument -k requires a non-empty value"}, {args: []string{"-k", "-m"}, wantErr: `pytest argument -k requires a value, not "-m": pass it in the next --pytest-arg`}, {args: []string{"-m", "--maxfail=1"}, wantErr: `pytest argument -m requires a value, not "--maxfail=1": pass it in the next --pytest-arg`}, {args: []string{"-k"}, wantErr: "pytest argument -k requires a value: pass it in the next --pytest-arg"},