Kettle add new fields to schema and BQ tables

kubernetes · Jul 23, 2020 · efcc2e8 · efcc2e8
1 parent 0b3a817
commit efcc2e8
Show file tree

Hide file tree

Showing 2 changed files with 53 additions and 36 deletions.
diff --git a/kettle/README.md b/kettle/README.md
@@ -63,6 +63,10 @@ kubectl logs -f $(kubectl get pod -l app=kettle -oname)
 
 It might take a couple of hours to be fully functional and start updating BigQuery. You can always go back to the [Gubernator BigQuery page](https://bigquery.cloud.google.com/table/k8s-gubernator:build.all?pli=1&tab=details) and check to see if data collection has resumed.  Backfill should happen automatically.
 
+#### Adding Fields
+
+To add fields to the BQ table, Visit the [k8s-gubernator:build BigQuery dataset](https://bigquery.cloud.google.com/dataset/k8s-gubernator:build) and Select the table (Ex. Build > All). Schema -> Edit Schema -> Add field. As well as update [schema.json](./schema.json)
+
 # Known Issues
 
 - Occasionally data from Kettle stops updating, we suspect this is due to a transient hang when contacting GCS ([#8800](https://github.com/kubernetes/test-infra/issues/8800)). If this happens, [restart kettle](#restarting)

diff --git a/kettle/schema.json b/kettle/schema.json
@@ -12,51 +12,52 @@
       "mode" : "NULLABLE"
    },
    {
+      "name" : "finished",
       "description" : "build end time",
       "type" : "TIMESTAMP",
-      "mode" : "NULLABLE",
-      "name" : "finished"
+      "mode" : "NULLABLE"
    },
    {
-      "description" : "Whether the build succeeded.",
-      "mode" : "NULLABLE",
+      "name" : "passed",
       "type" : "BOOLEAN",
-      "name" : "passed"
+      "description" : "Whether the build succeeded.",
+      "mode" : "NULLABLE"
    },
    {
-      "description" : "(DEPRECATED: use passed) \"SUCCESS\" \"FAILURE\" \"ABORTED\", etc",
-      "mode" : "NULLABLE",
+      "name" : "result",
       "type" : "STRING",
-      "name" : "result"
+      "description" : "(DEPRECATED: use passed) \"SUCCESS\" \"FAILURE\" \"ABORTED\", etc",
+      "mode" : "NULLABLE"
    },
    {
       "name" : "version",
-      "mode" : "NULLABLE",
-      "type" : "STRING"
+      "type" : "STRING",
+      "description" : "Version of kubernetes project",
+      "mode" : "NULLABLE"
    },
    {
-      "description" : "GCS path with build results",
-      "mode" : "NULLABLE",
+      "name" : "path",
       "type" : "STRING",
-      "name" : "path"
+      "description" : "GCS path with build results",
+      "mode" : "NULLABLE"
    },
    {
-      "description" : "Job name.",
-      "mode" : "NULLABLE",
+      "name" : "job",
       "type" : "STRING",
-      "name" : "job"
+      "description" : "Job name.",
+      "mode" : "NULLABLE"
    },
    {
-      "description" : "Build number.",
-      "mode" : "NULLABLE",
+      "name" : "number",
       "type" : "INTEGER",
-      "name" : "number"
+      "description" : "Build number.",
+      "mode" : "NULLABLE"
    },
    {
       "name": "metadata",
-      "mode" : "REPEATED",
       "type" : "RECORD",
       "description": "extra build information",
+      "mode" : "REPEATED",
       "fields": [
          {
             "mode" : "NULLABLE",
@@ -76,39 +77,39 @@
       "name" : "test",
       "fields" : [
          {
-            "description" : "",
-            "mode" : "NULLABLE",
+            "name" : "name",
             "type" : "STRING",
-            "name" : "name"
+            "description" : "",
+            "mode" : "NULLABLE"
          },
          {
-            "mode" : "NULLABLE",
-            "description" : "Elapsed test time.",
+            "name" : "time",
             "type" : "FLOAT",
-            "name" : "time"
+            "description" : "Elapsed test time.",
+            "mode" : "NULLABLE"
          },
          {
             "name" : "failed",
+            "type" : "BOOLEAN",
             "description" : "Did this test fail?",
-            "mode" : "NULLABLE",
-            "type" : "BOOLEAN"
+            "mode" : "NULLABLE"
          },
          {
             "name" : "failure_text",
+            "type" : "STRING",
             "description" : "Failure text (if test failed).",
-            "mode" : "NULLABLE",
-            "type" : "STRING"
+            "mode" : "NULLABLE"
          }
       ],
+      "type" : "RECORD",
       "description" : "Test results from individual JUnit files.",
-      "mode" : "REPEATED",
-      "type" : "RECORD"
+      "mode" : "REPEATED"
    },
    {
       "name" : "tests_run",
-      "mode" : "NULLABLE",
+      "type" : "INTEGER",
       "description" : "Number of tests run.",
-      "type" : "INTEGER"
+      "mode" : "NULLABLE"
    },
    {
       "name" : "tests_failed",
@@ -118,8 +119,20 @@
    },
    {
       "name" : "executor",
-      "mode" : "NULLABLE",
       "type" : "STRING",
-      "description": "Hostname of machine that ran the test"
+      "description": "Hostname of machine that ran the test",
+      "mode" : "NULLABLE"
+   },
+   {
+      "name" : "repos",
+      "type" : "STRING",
+      "description": "Branch and Hash for all included repos",
+      "mode" : "NULLABLE"
+   },
+   {
+      "name" : "repo_commit",
+      "type" : "STRING",
+      "description": "Commit Hash",
+      "mode" : "NULLABLE"
    }
 ]