Fix legacy filename migration in bundle generators

acceptance/bundle/generate/legacy_filename_migration/databricks.yml

@@ -0,0 +1,2 @@
+bundle:
+  name: legacy_filename_migration

acceptance/bundle/generate/legacy_filename_migration/output.txt

@@ -0,0 +1,28 @@
+
+=== job legacy filename
+>>> [CLI] bundle generate job --existing-job-id 1234 --key test_job --config-dir resources
+Job configuration successfully saved to resources/test_job.job.yml
+Warning: Generated configuration is not included in the bundle
+
+The file resources/test_job.job.yml is not matched by any pattern in the 'include' section of databricks.yml,
+so it will not be deployed. Add a matching entry to the 'include' section, for example:
+
+include:
+  - resources/*.yml
+
+
+=== pipeline legacy filename
+>>> [CLI] bundle generate pipeline --existing-pipeline-id 1234 --key test_pipeline --config-dir resources
+Pipeline configuration successfully saved to resources/test_pipeline.pipeline.yml
+Warning: Generated configuration is not included in the bundle
+
+The file resources/test_pipeline.pipeline.yml is not matched by any pattern in the 'include' section of databricks.yml,
+so it will not be deployed. Add a matching entry to the 'include' section, for example:
+
+include:
+  - resources/*.yml
+
+
+=== existing typed filename preserves both files
+>>> musterr [CLI] bundle generate job --existing-job-id 1234 --key test_job --config-dir resources
+Error: resources/test_job.job.yml already exists. Use --force to overwrite

acceptance/bundle/generate/legacy_filename_migration/script

@@ -0,0 +1,20 @@
+mkdir resources
+
+title "job legacy filename"
+echo legacy-job > resources/test_job.yml
+trace $CLI bundle generate job --existing-job-id 1234 --key test_job --config-dir resources
+test ! -e resources/test_job.yml
+grep -q "name: test-job" resources/test_job.job.yml
+
+title "pipeline legacy filename"
+echo legacy-pipeline > resources/test_pipeline.yml
+trace $CLI bundle generate pipeline --existing-pipeline-id 1234 --key test_pipeline --config-dir resources
+test ! -e resources/test_pipeline.yml
+grep -q "name: test-pipeline" resources/test_pipeline.pipeline.yml
+
+title "existing typed filename preserves both files"
+echo legacy-job > resources/test_job.yml
+echo typed-job > resources/test_job.job.yml
+trace musterr $CLI bundle generate job --existing-job-id 1234 --key test_job --config-dir resources
+test "$(cat resources/test_job.yml)" = legacy-job
+test "$(cat resources/test_job.job.yml)" = typed-job

acceptance/bundle/generate/legacy_filename_migration/test.toml

@@ -0,0 +1,24 @@
+Ignore = ["resources/"]
+
+[[Server]]
+Pattern = "GET /api/2.2/jobs/get"
+Response.Body = '''
+{
+  "job_id": 1234,
+  "settings": {
+    "name": "test-job"
+  }
+}
+'''
+
+[[Server]]
+Pattern = "GET /api/2.0/pipelines/1234"
+Response.Body = '''
+{
+  "pipeline_id": "1234",
+  "name": "test-pipeline",
+  "spec": {
+    "name": "test-pipeline"
+  }
+}
+'''

cmd/bundle/generate/job.go

@@ -133,14 +133,6 @@ After generation, you can deploy this job to other targets using:
 		oldFilename := filepath.Join(configDir, jobKey+".yml")
 		filename := filepath.Join(configDir, jobKey+".job.yml")
 
-		// User might continuously run generate command to update their bundle jobs with any changes made in Databricks UI.
-		// Due to changing in the generated file names, we need to first rename existing resource file to the new name.
-		// Otherwise users can end up with duplicated resources.
-		err = os.Rename(oldFilename, filename)
-		if err != nil && !errors.Is(err, fs.ErrNotExist) {
-			return fmt.Errorf("failed to rename file %s. DABs uses the resource type as a sub-extension for generated content, please rename it to %s, err: %w", oldFilename, filename, err)
-		}
-
 		saver := yamlsaver.NewSaverWithStyle(map[string]yaml.Style{
 			// Including all JobSettings and nested fields which are map[string]string type
 			"spark_conf":  yaml.DoubleQuotedStyle,
@@ -152,6 +144,13 @@ After generation, you can deploy this job to other targets using:
 			return err
 		}
 
+		// Remove the legacy filename only after the new file has been saved.
+		// Otherwise SaveAsYAML sees the renamed file and requires --force.
+		err = os.Remove(oldFilename)
+		if err != nil && !errors.Is(err, fs.ErrNotExist) {
+			return fmt.Errorf("failed to remove legacy generated file %s: %w", oldFilename, err)
+		}
+
 		cmdio.LogString(ctx, "Job configuration successfully saved to "+filepath.ToSlash(filename))
 
 		warnIfNotIncluded(ctx, b, filename)

cmd/bundle/generate/pipeline.go

@@ -128,14 +128,6 @@ like catalogs, schemas, and compute configurations per target.`,
 		oldFilename := filepath.Join(configDir, pipelineKey+".yml")
 		filename := filepath.Join(configDir, pipelineKey+".pipeline.yml")
 
-		// User might continuously run generate command to update their bundle jobs with any changes made in Databricks UI.
-		// Due to changing in the generated file names, we need to first rename existing resource file to the new name.
-		// Otherwise users can end up with duplicated resources.
-		err = os.Rename(oldFilename, filename)
-		if err != nil && !errors.Is(err, fs.ErrNotExist) {
-			return fmt.Errorf("failed to rename file %s. DABs uses the resource type as a sub-extension for generated content, please rename it to %s, err: %w", oldFilename, filename, err)
-		}
-
 		saver := yamlsaver.NewSaverWithStyle(
 			// Including all CreatePipeline and nested fields which are map[string]string type
 			map[string]yaml.Style{
@@ -149,6 +141,13 @@ like catalogs, schemas, and compute configurations per target.`,
 			return err
 		}
 
+		// Remove the legacy filename only after the new file has been saved.
+		// Otherwise SaveAsYAML sees the renamed file and requires --force.
+		err = os.Remove(oldFilename)
+		if err != nil && !errors.Is(err, fs.ErrNotExist) {
+			return fmt.Errorf("failed to remove legacy generated file %s: %w", oldFilename, err)
+		}
+
 		cmdio.LogString(ctx, "Pipeline configuration successfully saved to "+filepath.ToSlash(filename))
 
 		warnIfNotIncluded(ctx, b, filename)