WIP terraform engine removal

.github/workflows/push.yml

@@ -111,16 +111,14 @@ jobs:
             runner:
               labels: macos-latest
 
-        # Each cell runs the suite once. envfilter picks the variants it covers: without the
-        # recording filter, the direct cell would run every bundle test twice, since
-        # bundle/test.toml matrixes deployment-history recording on top of the engine.
+        # Each cell runs the suite once. envfilter picks the variants it covers: the plain
+        # cell excludes the deployment-history recording variant (DMS=true), which the
+        # directdms cell covers, so bundle/test.toml's DMS matrix does not run every test twice.
         deployment:
-          - name: "terraform"
-            envfilter: "DATABRICKS_BUNDLE_ENGINE=terraform"
           - name: "direct"
-            envfilter: "DATABRICKS_BUNDLE_ENGINE=direct,DMS="
+            envfilter: "DMS="
           - name: "directdms"
-            envfilter: "DATABRICKS_BUNDLE_ENGINE=direct,DMS=true"
+            envfilter: "DMS=true"
 
         # Include "event_name" in the matrix so we can include/exclude based on it.
         event:

.nextchanges/notable-changes/remove-terraform-engine.md

@@ -0,0 +1 @@
+* Remove the Terraform deployment engine. Bundles pinning `engine: terraform` (or `DATABRICKS_BUNDLE_ENGINE=terraform`) now error and should either move to the direct engine or downgrade to CLI v1.18.x. ([#6746](https://github.com/databricks/cli/pull/6746))

NOTICE

@@ -25,22 +25,10 @@ hashicorp/go-version - https://github.com/hashicorp/go-version
 Copyright 2014 HashiCorp, Inc.
 License - https://github.com/hashicorp/go-version/blob/main/LICENSE
 
-hashicorp/hc-install - https://github.com/hashicorp/hc-install
-Copyright 2020 HashiCorp, Inc.
-License - https://github.com/hashicorp/hc-install/blob/main/LICENSE
-
-hashicorp/terraform-exec - https://github.com/hashicorp/terraform-exec
-Copyright 2020 HashiCorp, Inc.
-License - https://github.com/hashicorp/terraform-exec/blob/main/LICENSE
-
 hashicorp/terraform-json - https://github.com/hashicorp/terraform-json
 Copyright 2019 HashiCorp, Inc.
 License - https://github.com/hashicorp/terraform-json/blob/main/LICENSE
 
-hashicorp/terraform - https://github.com/hashicorp/terraform
-Copyright 2014 HashiCorp, Inc.
-License - https://github.com/hashicorp/terraform/blob/v1.5.5/LICENSE
-
 ---
 
 This Software contains code from the following open source projects, licensed under the BSD (2-clause) license:

acceptance/apps/deploy/bundle-no-args-with-flags/test.toml

@@ -4,7 +4,6 @@ Ignore = [
     '.databricks',
 ]
 
-EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"]
 
 [[Server]]
 Pattern = "POST /api/2.0/apps/myapp/deployments"

acceptance/apps/deploy/bundle-no-args/test.toml

@@ -4,7 +4,6 @@ Ignore = [
     '.databricks',
 ]
 
-EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"]
 
 [[Server]]
 Pattern = "POST /api/2.0/apps/myapp/deployments"

acceptance/bin/print_state.py

@@ -220,9 +220,10 @@ def main():
     for filename in get_state_files(args.target, args.backup):
         if not os.path.exists(filename):
             continue
-        # Recording only applies to the direct engine, so a terraform run prints the file as-is.
+        # Recording only applies to direct-engine state; a terraform state file
+        # (e.g. one awaiting migration) is printed as-is.
         recording = os.environ.get("DATABRICKS_BUNDLE_DEPLOYMENT_HISTORY") == "true"
-        terraform = os.environ.get("DATABRICKS_BUNDLE_ENGINE") == "terraform"
+        terraform = "terraform.tfstate" in filename
         if recording and not terraform and not args.no_dms:
             print_recorded_state(filename, args.target)
         else:

acceptance/bin/read_state.py

@@ -79,7 +79,5 @@ def print_resource_recorded(group, name, *attrs):
 
 if os.environ.get("DATABRICKS_BUNDLE_DEPLOYMENT_HISTORY") == "true":
     print_resource_recorded(*sys.argv[1:])
-elif os.environ.get("DATABRICKS_BUNDLE_ENGINE", "").startswith("direct"):
-    print_resource_direct(*sys.argv[1:])
 else:
-    print_resource_terraform(*sys.argv[1:])
+    print_resource_direct(*sys.argv[1:])

acceptance/bundle/apps/compute_size/script

@@ -17,4 +17,4 @@ sed -i.bak 's/compute_size: LARGE/compute_size: MEDIUM/' databricks.yml
 {
   trace errcode $CLI bundle deploy
   trace $CLI apps get "app-$UNIQUE_NAME" | jq '{compute_size: .compute_size}'
-} &> out.update.$DATABRICKS_BUNDLE_ENGINE.txt
+} &> out.update.txt

acceptance/bundle/apps/delete_deleting/test.toml

@@ -2,6 +2,5 @@ Cloud = false
 RecordRequests = false
 
 # Saved-plan deploy (`--plan`) and the IsGone gone-detection are direct-engine only.
-EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"]
 
 Ignore = [".databricks", "databricks.yml", "plan_delete.json"]

acceptance/bundle/apps/job_permissions/script

@@ -13,7 +13,7 @@ has_manage_run() {
 }
 
 title "After first deploy"
-trace has_manage_run > out.after_first_deploy.$DATABRICKS_BUNDLE_ENGINE.txt
+trace has_manage_run > out.after_first_deploy.txt
 # This is flaky on direct, because there are (at least) two ways to complete deployment:
 # sequence 1:
 #  job is deployed
@@ -24,7 +24,7 @@ trace has_manage_run > out.after_first_deploy.$DATABRICKS_BUNDLE_ENGINE.txt
 #  app is deployed, updates job permissions
 #  job permissions are deployed
 # (It does not appear flaky on TF, maybe there is enough delay in one of the resources)
-rm -f out.after_first_deploy.direct.txt
+rm -f out.after_first_deploy.txt
 
 title "After second deploy"
 trace $CLI bundle deploy -qq

acceptance/bundle/apps/job_permissions/test.toml

@@ -1,4 +1,2 @@
 Cloud = true
 RecordRequests = false
-
-EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"]

acceptance/bundle/apps/source_code_path_update/test.toml

@@ -4,7 +4,6 @@ RecordRequests = false
 
 # This regression is in the direct engine only (the bug is in InitForApply
 # which is only called for bundle deploy --plan on the direct engine).
-EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"]
 
 Ignore = [".databricks", "databricks.yml", "databricks.yml.bak", "plan.json", "LOG.*"]
 

acceptance/bundle/artifacts/ai_runtime_code_source/test.toml

@@ -6,4 +6,3 @@ Ignore = [
 
 # The Terraform provider dropped `code_source_path` from `ai_runtime_task` in
 # v1.122.0, so this feature is only exercisable via the direct engine.
-EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"]

acceptance/bundle/artifacts/whl_dynamic/script

@@ -6,7 +6,7 @@ cp -r $TESTDIR/../whl_prebuilt_multiple/dist/lib/other_test_code-0.0.1-py3-none-
 
 trace $CLI bundle validate -o json | nostamp | jq .artifacts
 
-$CLI bundle plan -o json | nostamp > out.plan_create.$DATABRICKS_BUNDLE_ENGINE.json
+$CLI bundle plan -o json | nostamp > out.plan_create.json
 trace $CLI bundle deploy
 
 title "There are 2 original wheels and 2 patched ones"
@@ -25,7 +25,7 @@ rm "$OUT_REQUESTS"
 
 title "Updating the local wheel and deploying again\n"
 touch my_test_code/src/new_module.py
-$CLI bundle plan -o json | nostamp > out.plan_update.$DATABRICKS_BUNDLE_ENGINE.json
+$CLI bundle plan -o json | nostamp > out.plan_update.json
 trace $CLI bundle deploy
 
 title "Verify contents, it should now have new_module.py"