Cloudflare update secrets

[kwindau]

Description

Switch from using Airflow environment variables to using secrets from Google Secrets Manager instead

Related Tickets & Documents

• N/A

Reviewer, please follow this checklist

┆Issue is synchronized with this Jira Task

[dataops-ci-bot]

Integration report for "use new secret from GSM"

sql.diff

Click to expand!

diff -bur --no-dereference --new-file /tmp/workspace/main-generated-sql/dags/bqetl_cloudflare_browser_market_share.py /tmp/workspace/generated-sql/dags/bqetl_cloudflare_browser_market_share.py
--- /tmp/workspace/main-generated-sql/dags/bqetl_cloudflare_browser_market_share.py	2024-09-28 23:14:58.000000000 +0000
+++ /tmp/workspace/generated-sql/dags/bqetl_cloudflare_browser_market_share.py	2024-09-28 23:17:19.000000000 +0000
@@ -4,6 +4,7 @@
 from airflow.sensors.external_task import ExternalTaskMarker
 from airflow.sensors.external_task import ExternalTaskSensor
 from airflow.utils.task_group import TaskGroup
+from airflow.providers.cncf.kubernetes.secret import Secret
 import datetime
 from operators.gcp_container_operator import GKEPodOperator
 from utils.constants import ALLOWED_STATES, FAILED_STATES
@@ -28,6 +29,13 @@
 * repo/bigquery-etl
 """
 
+cloudflare_derived__browser_usage__v1_CLOUDFLARE_AUTH_TOKEN = Secret(
+    deploy_type="env",
+    deploy_target="SECRET",
+    secret="gke-secrets",
+    key="CLOUDFLARE_AUTH_TOKEN",
+)
+
 
 default_args = {
     "owner": "[email protected]",
@@ -57,13 +65,11 @@
             "python",
             "sql/moz-fx-data-shared-prod/cloudflare_derived/browser_usage_v1/query.py",
         ]
-        + [
-            "--date",
-            "{{ds}}",
-            "--cloudflare_api_token",
-            "{{ var.value.cloudflare_auth_token}}",
-        ],
+        + ["--date", "{{ds}}"],
         image="gcr.io/moz-fx-data-airflow-prod-88e0/bigquery-etl:latest",
         owner="[email protected]",
         email=["[email protected]"],
+        secrets=[
+            cloudflare_derived__browser_usage__v1_CLOUDFLARE_AUTH_TOKEN,
+        ],
     )
diff -bur --no-dereference --new-file /tmp/workspace/main-generated-sql/dags/bqetl_cloudflare_device_market_share.py /tmp/workspace/generated-sql/dags/bqetl_cloudflare_device_market_share.py
--- /tmp/workspace/main-generated-sql/dags/bqetl_cloudflare_device_market_share.py	2024-09-28 23:14:58.000000000 +0000
+++ /tmp/workspace/generated-sql/dags/bqetl_cloudflare_device_market_share.py	2024-09-28 23:17:19.000000000 +0000
@@ -4,6 +4,7 @@
 from airflow.sensors.external_task import ExternalTaskMarker
 from airflow.sensors.external_task import ExternalTaskSensor
 from airflow.utils.task_group import TaskGroup
+from airflow.providers.cncf.kubernetes.secret import Secret
 import datetime
 from operators.gcp_container_operator import GKEPodOperator
 from utils.constants import ALLOWED_STATES, FAILED_STATES
@@ -28,6 +29,13 @@
 * repo/bigquery-etl
 """
 
+cloudflare_derived__device_usage__v1_CLOUDFLARE_AUTH_TOKEN = Secret(
+    deploy_type="env",
+    deploy_target="SECRET",
+    secret="gke-secrets",
+    key="CLOUDFLARE_AUTH_TOKEN",
+)
+
 
 default_args = {
     "owner": "[email protected]",
@@ -57,13 +65,11 @@
             "python",
             "sql/moz-fx-data-shared-prod/cloudflare_derived/device_usage_v1/query.py",
         ]
-        + [
-            "--date",
-            "{{ds}}",
-            "--cloudflare_api_token",
-            "{{ var.value.cloudflare_auth_token}}",
-        ],
+        + ["--date", "{{ds}}"],
         image="gcr.io/moz-fx-data-airflow-prod-88e0/bigquery-etl:latest",
         owner="[email protected]",
         email=["[email protected]"],
+        secrets=[
+            cloudflare_derived__device_usage__v1_CLOUDFLARE_AUTH_TOKEN,
+        ],
     )
diff -bur --no-dereference --new-file /tmp/workspace/main-generated-sql/dags/bqetl_cloudflare_os_market_share.py /tmp/workspace/generated-sql/dags/bqetl_cloudflare_os_market_share.py
--- /tmp/workspace/main-generated-sql/dags/bqetl_cloudflare_os_market_share.py	2024-09-28 23:14:58.000000000 +0000
+++ /tmp/workspace/generated-sql/dags/bqetl_cloudflare_os_market_share.py	2024-09-28 23:17:19.000000000 +0000
@@ -4,6 +4,7 @@
 from airflow.sensors.external_task import ExternalTaskMarker
 from airflow.sensors.external_task import ExternalTaskSensor
 from airflow.utils.task_group import TaskGroup
+from airflow.providers.cncf.kubernetes.secret import Secret
 import datetime
 from operators.gcp_container_operator import GKEPodOperator
 from utils.constants import ALLOWED_STATES, FAILED_STATES
@@ -28,6 +29,13 @@
 * repo/bigquery-etl
 """
 
+cloudflare_derived__os_usage__v1_CLOUDFLARE_AUTH_TOKEN = Secret(
+    deploy_type="env",
+    deploy_target="SECRET",
+    secret="gke-secrets",
+    key="CLOUDFLARE_AUTH_TOKEN",
+)
+
 
 default_args = {
     "owner": "[email protected]",
@@ -57,13 +65,11 @@
             "python",
             "sql/moz-fx-data-shared-prod/cloudflare_derived/os_usage_v1/query.py",
         ]
-        + [
-            "--date",
-            "{{ds}}",
-            "--cloudflare_api_token",
-            "{{ var.value.cloudflare_auth_token}}",
-        ],
+        + ["--date", "{{ds}}"],
         image="gcr.io/moz-fx-data-airflow-prod-88e0/bigquery-etl:latest",
         owner="[email protected]",
         email=["[email protected]"],
+        secrets=[
+            cloudflare_derived__os_usage__v1_CLOUDFLARE_AUTH_TOKEN,
+        ],
     )
diff -bur --no-dereference --new-file /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/cloudflare_derived/browser_usage_v1/metadata.yaml /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/cloudflare_derived/browser_usage_v1/metadata.yaml
--- /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/cloudflare_derived/browser_usage_v1/metadata.yaml	2024-09-28 23:11:45.000000000 +0000
+++ /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/cloudflare_derived/browser_usage_v1/metadata.yaml	2024-09-28 23:12:29.000000000 +0000
@@ -13,8 +13,11 @@
   arguments:
   - --date
   - '{{ds}}'
-  - --cloudflare_api_token
-  - '{{ var.value.cloudflare_auth_token}}'
+  secrets:
+  - deploy_target: SECRET
+    key: CLOUDFLARE_AUTH_TOKEN
+    deploy_type: env
+    secret: gke-secrets
 bigquery:
   time_partitioning:
     type: day
diff -bur --no-dereference --new-file /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/cloudflare_derived/browser_usage_v1/query.py /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/cloudflare_derived/browser_usage_v1/query.py
--- /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/cloudflare_derived/browser_usage_v1/query.py	2024-09-28 23:09:51.000000000 +0000
+++ /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/cloudflare_derived/browser_usage_v1/query.py	2024-09-28 23:10:02.000000000 +0000
@@ -6,6 +6,7 @@
 from argparse import ArgumentParser
 from google.cloud import bigquery
 from google.cloud import storage
+from airflow.providers.cncf.kubernetes.secret import Secret
 
 # Configs
 brwsr_usg_configs = {
@@ -75,6 +76,13 @@
     "errors_bq_stg_table": "moz-fx-data-shared-prod.cloudflare_derived.browser_errors_stg",
 }
 
+#Load the Cloudflare API Token
+cloudflare_api_token = Secret(
+    deploy_type="env",
+    deploy_target="SECRET",
+    secret="gke-secrets",
+    key="CLOUDFLARE_AUTH_TOKEN",
+)
 
 # Define a function to move a GCS object then delete the original
 def move_blob(bucket_name, blob_name, destination_bucket_name, destination_blob_name):
@@ -269,7 +277,7 @@
     """Call the API, save data to GCS, load to BQ staging, delete & load to BQ gold"""
     parser = ArgumentParser(description=__doc__)
     parser.add_argument("--date", required=True)
-    parser.add_argument("--cloudflare_api_token", required=True)
+    parser.add_argument("--cloudflare_api_token", default=cloudflare_api_token, required=True)
     parser.add_argument("--project", default=brwsr_usg_configs["gcp_project_id"])
     parser.add_argument("--dataset", default="cloudflare_derived")
 
diff -bur --no-dereference --new-file /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/cloudflare_derived/device_usage_v1/metadata.yaml /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/cloudflare_derived/device_usage_v1/metadata.yaml
--- /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/cloudflare_derived/device_usage_v1/metadata.yaml	2024-09-28 23:11:45.000000000 +0000
+++ /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/cloudflare_derived/device_usage_v1/metadata.yaml	2024-09-28 23:12:29.000000000 +0000
@@ -12,8 +12,11 @@
   arguments:
   - --date
   - '{{ds}}'
-  - --cloudflare_api_token
-  - '{{ var.value.cloudflare_auth_token}}'
+  secrets:
+  - deploy_target: SECRET
+    key: CLOUDFLARE_AUTH_TOKEN
+    deploy_type: env
+    secret: gke-secrets
 bigquery:
   time_partitioning:
     type: day
diff -bur --no-dereference --new-file /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/cloudflare_derived/device_usage_v1/query.py /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/cloudflare_derived/device_usage_v1/query.py
--- /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/cloudflare_derived/device_usage_v1/query.py	2024-09-28 23:09:51.000000000 +0000
+++ /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/cloudflare_derived/device_usage_v1/query.py	2024-09-28 23:10:02.000000000 +0000
@@ -6,6 +6,7 @@
 from argparse import ArgumentParser
 from google.cloud import bigquery
 from google.cloud import storage
+from airflow.providers.cncf.kubernetes.secret import Secret
 
 # Configs
 device_usg_configs = {
@@ -62,6 +63,14 @@
     "errors_bq_stg_table": "moz-fx-data-shared-prod.cloudflare_derived.device_errors_stg",
 }
 
+#Load the Cloudflare API Token
+cloudflare_api_token = Secret(
+    deploy_type="env",
+    deploy_target="SECRET",
+    secret="gke-secrets",
+    key="CLOUDFLARE_AUTH_TOKEN",
+)
+
 
 # Define a function to move a GCS object then delete the original
 def move_blob(bucket_name, blob_name, destination_bucket_name, destination_blob_name):
@@ -285,7 +294,7 @@
     """Call the API, save data to GCS, load to BQ staging, delete & load to BQ gold"""
     parser = ArgumentParser(description=__doc__)
     parser.add_argument("--date", required=True)
-    parser.add_argument("--cloudflare_api_token", required=True)
+    parser.add_argument("--cloudflare_api_token", default=cloudflare_api_token, required=True)
     parser.add_argument("--project", default="moz-fx-data-shared-prod")
     parser.add_argument("--dataset", default="cloudflare_derived")
 
diff -bur --no-dereference --new-file /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/cloudflare_derived/os_usage_v1/metadata.yaml /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/cloudflare_derived/os_usage_v1/metadata.yaml
--- /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/cloudflare_derived/os_usage_v1/metadata.yaml	2024-09-28 23:11:45.000000000 +0000
+++ /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/cloudflare_derived/os_usage_v1/metadata.yaml	2024-09-28 23:12:29.000000000 +0000
@@ -13,8 +13,11 @@
   arguments:
   - --date
   - '{{ds}}'
-  - --cloudflare_api_token
-  - '{{ var.value.cloudflare_auth_token}}'
+  secrets:
+  - deploy_target: SECRET
+    key: CLOUDFLARE_AUTH_TOKEN
+    deploy_type: env
+    secret: gke-secrets
 bigquery:
   time_partitioning:
     type: day
diff -bur --no-dereference --new-file /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/cloudflare_derived/os_usage_v1/query.py /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/cloudflare_derived/os_usage_v1/query.py
--- /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/cloudflare_derived/os_usage_v1/query.py	2024-09-28 23:09:51.000000000 +0000
+++ /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/cloudflare_derived/os_usage_v1/query.py	2024-09-28 23:10:02.000000000 +0000
@@ -6,6 +6,7 @@
 from argparse import ArgumentParser
 from google.cloud import bigquery
 from google.cloud import storage
+from airflow.providers.cncf.kubernetes.secret import Secret
 
 # Configurations
 os_usg_configs = {
@@ -63,6 +64,13 @@
     "errors_bq_stg_table": "moz-fx-data-shared-prod.cloudflare_derived.os_errors_stg",
 }
 
+#Load the Cloudflare API Token
+cloudflare_api_token = Secret(
+    deploy_type="env",
+    deploy_target="SECRET",
+    secret="gke-secrets",
+    key="CLOUDFLARE_AUTH_TOKEN",
+)
 
 # Define a function to move a GCS object then delete the original
 def move_blob(bucket_name, blob_name, destination_bucket_name, destination_blob_name):
@@ -233,7 +241,7 @@
     """Call the API, save data to GCS, load to BQ staging, delete & load to BQ gold"""
     parser = ArgumentParser(description=__doc__)
     parser.add_argument("--date", required=True)
-    parser.add_argument("--cloudflare_api_token", required=True)
+    parser.add_argument("--cloudflare_api_token", default=cloudflare_api_token, required=True)
     parser.add_argument("--project", default="moz-fx-data-shared-prod")
     parser.add_argument("--dataset", default="cloudflare_derived")
 

Link to full diff