Added logic to fully resolve targets (with prefixes/suffixes) when up…

…dating the Data Preparation YAML. (#1866) * Added logic to fully resolve targets (with prefixes/suffixes) when updating the Data Preparation YAML. * Added logic for target finalization.
dataform-co · Nov 1, 2024 · 136a990 · 136a990
1 parent e16717f
commit 136a990
Show file tree

Hide file tree

Showing 3 changed files with 51 additions and 22 deletions.
diff --git a/core/actions/data_preparation.ts b/core/actions/data_preparation.ts
@@ -41,8 +41,12 @@ export class DataPreparation extends ActionBuilder<dataform.DataPreparation> {
     const targets = this.getTargets(dataPreparationAsJson as {
       [key: string]: any;
     });
-    this.proto.targets = targets.map(target =>
-      this.applySessionToTarget(target, session.projectConfig, config.filename, true)
+    const resolvedTargets = targets.map(target =>
+        this.applySessionToTarget(target, session.projectConfig, config.filename, true)
+    )
+    // Finalize list of targets.
+    this.proto.targets = resolvedTargets.map(target =>
+        this.finalizeTarget(target)
     );
     this.proto.canonicalTargets = targets.map(target =>
       this.applySessionToTarget(target, session.canonicalProjectConfig)
@@ -54,7 +58,7 @@ export class DataPreparation extends ActionBuilder<dataform.DataPreparation> {
 
     // Set the unique target key as the first target defined.
     // TODO: Remove once multiple targets are supported.
-    this.proto.target = this.proto.targets[0];
+    this.proto.target = resolvedTargets[0];
     this.proto.canonicalTarget = this.proto.canonicalTargets[0];
 
     this.proto.tags = config.tags;
@@ -161,15 +165,21 @@ export class DataPreparation extends ActionBuilder<dataform.DataPreparation> {
             this.session.projectConfig)
     // Convert resolved target into a Data Preparation Table Reference
     let resolvedTableReference : {[key: string]: string} = {
-      table: resolvedTarget.name,
+      table: this.session.finalizeName(resolvedTarget.name),
     }
 
     // Ensure project and dataset field are added in order
     if (resolvedTarget.schema) {
-      resolvedTableReference = { dataset: resolvedTarget.schema, ...resolvedTableReference }
+      resolvedTableReference = {
+        dataset: this.session.finalizeSchema(resolvedTarget.schema),
+        ...resolvedTableReference
+      }
     }
     if (resolvedTarget.database) {
-      resolvedTableReference = { project: resolvedTarget.database, ...resolvedTableReference }
+      resolvedTableReference = {
+        project: this.session.finalizeDatabase(resolvedTarget.database),
+        ...resolvedTableReference
+      }
     }
     return resolvedTableReference;
   }

diff --git a/core/actions/index.ts b/core/actions/index.ts
@@ -50,6 +50,18 @@ export abstract class ActionBuilder<T> {
     return target;
   }
 
+  public finalizeTarget(
+    targetFromConfig: dataform.Target
+  ): dataform.Target {
+    return dataform.Target.create({
+      name: this.session.finalizeName(targetFromConfig.name),
+      schema: targetFromConfig.schema ?
+          this.session.finalizeSchema(targetFromConfig.schema) : undefined,
+      database: targetFromConfig.database ?
+          this.session.finalizeDatabase(targetFromConfig.database) : undefined
+    })
+  }
+
   /** Retrieves the filename from the config. */
   public abstract getFileName(): string;
 

diff --git a/core/main_test.ts b/core/main_test.ts
@@ -955,7 +955,14 @@ nodes:
     });
 
     test(`data preparations resolves compilation overrides before encoding`, () => {
-      const projectDir = createSimpleDataPreparationProject();
+      const projectDir = createSimpleDataPreparationProject(`
+defaultProject: defaultProject
+defaultDataset: defaultDataset
+defaultLocation: US
+projectSuffix: projectSuffix
+datasetSuffix: datasetSuffix
+namePrefix: tablePrefix
+`);
       const dataPreparationYaml = `
 configuration:
   errorTable:
@@ -1013,16 +1020,16 @@ nodes:
       const resolvedYaml = `
 configuration:
   errorTable:
-    project: defaultProject
-    dataset: defaultDataset
-    table: error
+    project: defaultProject_projectSuffix
+    dataset: defaultDataset_datasetSuffix
+    table: tablePrefix_error
 nodes:
 - id: node1
   source:
     table:
-      project: defaultProject
-      dataset: defaultDataset
-      table: src
+      project: defaultProject_projectSuffix
+      dataset: defaultDataset_datasetSuffix
+      table: tablePrefix_src
   generated:
     sourceGenerated:
       sourceSchema:
@@ -1041,9 +1048,9 @@ nodes:
     nodeId: node1
   destination:
     table:
-      project: defaultProject
-      dataset: defaultDataset
-      table: dest
+      project: defaultProject_projectSuffix
+      dataset: defaultDataset_datasetSuffix
+      table: tablePrefix_dest
   generated:
     sourceGenerated:
       sourceSchema:
@@ -1072,9 +1079,9 @@ nodes:
           asPlainObject([
             {
               target: {
-                database: "defaultProject",
-                schema: "defaultDataset",
-                name: "dest"
+                database: "defaultProject_projectSuffix",
+                schema: "defaultDataset_datasetSuffix",
+                name: "tablePrefix_dest"
               },
               canonicalTarget: {
                 database: "defaultProject",
@@ -1083,9 +1090,9 @@ nodes:
               },
               targets: [
                 {
-                  database: "defaultProject",
-                  schema: "defaultDataset",
-                  name: "dest"
+                  database: "defaultProject_projectSuffix",
+                  schema: "defaultDataset_datasetSuffix",
+                  name: "tablePrefix_dest"
                 }
               ],
               canonicalTargets: [