diff --git a/acceptance/bundle/generate/legacy_filename_migration/databricks.yml b/acceptance/bundle/generate/legacy_filename_migration/databricks.yml new file mode 100644 index 00000000000..f944b8583f0 --- /dev/null +++ b/acceptance/bundle/generate/legacy_filename_migration/databricks.yml @@ -0,0 +1,2 @@ +bundle: + name: legacy_filename_migration diff --git a/acceptance/bundle/generate/legacy_filename_migration/out.test.toml b/acceptance/bundle/generate/legacy_filename_migration/out.test.toml new file mode 100644 index 00000000000..f784a183258 --- /dev/null +++ b/acceptance/bundle/generate/legacy_filename_migration/out.test.toml @@ -0,0 +1,3 @@ +Local = true +Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/generate/legacy_filename_migration/output.txt b/acceptance/bundle/generate/legacy_filename_migration/output.txt new file mode 100644 index 00000000000..9d785024e47 --- /dev/null +++ b/acceptance/bundle/generate/legacy_filename_migration/output.txt @@ -0,0 +1,28 @@ + +=== job legacy filename +>>> [CLI] bundle generate job --existing-job-id 1234 --key test_job --config-dir resources +Job configuration successfully saved to resources/test_job.job.yml +Warning: Generated configuration is not included in the bundle + +The file resources/test_job.job.yml is not matched by any pattern in the 'include' section of databricks.yml, +so it will not be deployed. Add a matching entry to the 'include' section, for example: + +include: + - resources/*.yml + + +=== pipeline legacy filename +>>> [CLI] bundle generate pipeline --existing-pipeline-id 1234 --key test_pipeline --config-dir resources +Pipeline configuration successfully saved to resources/test_pipeline.pipeline.yml +Warning: Generated configuration is not included in the bundle + +The file resources/test_pipeline.pipeline.yml is not matched by any pattern in the 'include' section of databricks.yml, +so it will not be deployed. Add a matching entry to the 'include' section, for example: + +include: + - resources/*.yml + + +=== existing typed filename preserves both files +>>> musterr [CLI] bundle generate job --existing-job-id 1234 --key test_job --config-dir resources +Error: resources/test_job.job.yml already exists. Use --force to overwrite diff --git a/acceptance/bundle/generate/legacy_filename_migration/script b/acceptance/bundle/generate/legacy_filename_migration/script new file mode 100644 index 00000000000..b5efee146ab --- /dev/null +++ b/acceptance/bundle/generate/legacy_filename_migration/script @@ -0,0 +1,20 @@ +mkdir resources + +title "job legacy filename" +echo legacy-job > resources/test_job.yml +trace $CLI bundle generate job --existing-job-id 1234 --key test_job --config-dir resources +test ! -e resources/test_job.yml +grep -q "name: test-job" resources/test_job.job.yml + +title "pipeline legacy filename" +echo legacy-pipeline > resources/test_pipeline.yml +trace $CLI bundle generate pipeline --existing-pipeline-id 1234 --key test_pipeline --config-dir resources +test ! -e resources/test_pipeline.yml +grep -q "name: test-pipeline" resources/test_pipeline.pipeline.yml + +title "existing typed filename preserves both files" +echo legacy-job > resources/test_job.yml +echo typed-job > resources/test_job.job.yml +trace musterr $CLI bundle generate job --existing-job-id 1234 --key test_job --config-dir resources +test "$(cat resources/test_job.yml)" = legacy-job +test "$(cat resources/test_job.job.yml)" = typed-job diff --git a/acceptance/bundle/generate/legacy_filename_migration/test.toml b/acceptance/bundle/generate/legacy_filename_migration/test.toml new file mode 100644 index 00000000000..d7a0a9b413e --- /dev/null +++ b/acceptance/bundle/generate/legacy_filename_migration/test.toml @@ -0,0 +1,24 @@ +Ignore = ["resources/"] + +[[Server]] +Pattern = "GET /api/2.2/jobs/get" +Response.Body = ''' +{ + "job_id": 1234, + "settings": { + "name": "test-job" + } +} +''' + +[[Server]] +Pattern = "GET /api/2.0/pipelines/1234" +Response.Body = ''' +{ + "pipeline_id": "1234", + "name": "test-pipeline", + "spec": { + "name": "test-pipeline" + } +} +''' diff --git a/cmd/bundle/generate/job.go b/cmd/bundle/generate/job.go index db6839ade98..ec9a13ee647 100644 --- a/cmd/bundle/generate/job.go +++ b/cmd/bundle/generate/job.go @@ -133,14 +133,6 @@ After generation, you can deploy this job to other targets using: oldFilename := filepath.Join(configDir, jobKey+".yml") filename := filepath.Join(configDir, jobKey+".job.yml") - // User might continuously run generate command to update their bundle jobs with any changes made in Databricks UI. - // Due to changing in the generated file names, we need to first rename existing resource file to the new name. - // Otherwise users can end up with duplicated resources. - err = os.Rename(oldFilename, filename) - if err != nil && !errors.Is(err, fs.ErrNotExist) { - return fmt.Errorf("failed to rename file %s. DABs uses the resource type as a sub-extension for generated content, please rename it to %s, err: %w", oldFilename, filename, err) - } - saver := yamlsaver.NewSaverWithStyle(map[string]yaml.Style{ // Including all JobSettings and nested fields which are map[string]string type "spark_conf": yaml.DoubleQuotedStyle, @@ -152,6 +144,13 @@ After generation, you can deploy this job to other targets using: return err } + // Remove the legacy filename only after the new file has been saved. + // Otherwise SaveAsYAML sees the renamed file and requires --force. + err = os.Remove(oldFilename) + if err != nil && !errors.Is(err, fs.ErrNotExist) { + return fmt.Errorf("failed to remove legacy generated file %s: %w", oldFilename, err) + } + cmdio.LogString(ctx, "Job configuration successfully saved to "+filepath.ToSlash(filename)) warnIfNotIncluded(ctx, b, filename) diff --git a/cmd/bundle/generate/pipeline.go b/cmd/bundle/generate/pipeline.go index cdde5984d40..8e428c6a314 100644 --- a/cmd/bundle/generate/pipeline.go +++ b/cmd/bundle/generate/pipeline.go @@ -128,14 +128,6 @@ like catalogs, schemas, and compute configurations per target.`, oldFilename := filepath.Join(configDir, pipelineKey+".yml") filename := filepath.Join(configDir, pipelineKey+".pipeline.yml") - // User might continuously run generate command to update their bundle jobs with any changes made in Databricks UI. - // Due to changing in the generated file names, we need to first rename existing resource file to the new name. - // Otherwise users can end up with duplicated resources. - err = os.Rename(oldFilename, filename) - if err != nil && !errors.Is(err, fs.ErrNotExist) { - return fmt.Errorf("failed to rename file %s. DABs uses the resource type as a sub-extension for generated content, please rename it to %s, err: %w", oldFilename, filename, err) - } - saver := yamlsaver.NewSaverWithStyle( // Including all CreatePipeline and nested fields which are map[string]string type map[string]yaml.Style{ @@ -149,6 +141,13 @@ like catalogs, schemas, and compute configurations per target.`, return err } + // Remove the legacy filename only after the new file has been saved. + // Otherwise SaveAsYAML sees the renamed file and requires --force. + err = os.Remove(oldFilename) + if err != nil && !errors.Is(err, fs.ErrNotExist) { + return fmt.Errorf("failed to remove legacy generated file %s: %w", oldFilename, err) + } + cmdio.LogString(ctx, "Pipeline configuration successfully saved to "+filepath.ToSlash(filename)) warnIfNotIncluded(ctx, b, filename)