-
Notifications
You must be signed in to change notification settings - Fork 0
/
Copy pathdatawagon-config.toml
124 lines (110 loc) · 4.36 KB
/
datawagon-config.toml
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
title = "CSV Source Files Configuration"
[file.claim_raw]
# example file: YouTube_SomeBrandName_M_20230601_claim_raw_v1-1.csv.tar.gz
is_enabled = true
storage_folder_name = "caravan/claim_raw"
table_name = "claim_raw"
select_file_name_base = "claim_raw"
exclude_file_name_base = "adj_claim_raw"
regex_pattern = 'YouTube_(.+)_M_(\d{8}|\d{6})'
regex_group_names = ["content_owner", "file_date_key"]
# file_date_key is special and it will be converted to a eom report_date_key when found (for now)
table_append_or_replace = "append"
[file.adj_claim_raw]
is_enabled = true
storage_folder_name = "caravan/adj_claim_raw"
table_name = "adj_claim_raw"
select_file_name_base = "adj_claim_raw"
regex_pattern = 'YouTube_(.+)_M_(\d{8}|\d{6})'
regex_group_names = ["content_owner", "file_date_key"]
table_append_or_replace = "append"
[file.asset_raw]
is_enabled = true
storage_folder_name = "caravan/asset_raw"
table_name = "asset_raw"
select_file_name_base = "asset_raw"
exclude_file_name_base = "adj_asset_raw"
regex_pattern = 'YouTube_(.+)_M_(\d{8}|\d{6})'
regex_group_names = ["content_owner", "file_date_key"]
table_append_or_replace = "append"
[file.adj_asset_raw]
is_enabled = true
storage_folder_name = "caravan/adj_asset_raw"
table_name = "adj_asset_raw"
select_file_name_base = "adj_asset_raw"
regex_pattern = 'YouTube_(.+)_M_(\d{8}|\d{6})'
regex_group_names = ["content_owner", "file_date_key"]
table_append_or_replace = "append"
[file.video_raw]
is_enabled = true
storage_folder_name = "caravan/video_raw"
table_name = "video_raw"
select_file_name_base = "video_raw"
exclude_file_name_base = "adj_video_raw"
regex_pattern = 'YouTube_(.+)_M_(\d{8}|\d{6})'
regex_group_names = ["content_owner", "file_date_key"]
table_append_or_replace = "append"
[file.adj_video_raw]
is_enabled = true
storage_folder_name = "caravan/adj_video_raw"
table_name = "adj_video_raw"
select_file_name_base = "adj_video_raw"
regex_pattern = 'YouTube_(.+)_M_(\d{8}|\d{6})'
regex_group_names = ["content_owner", "file_date_key"]
table_append_or_replace = "append"
[file.red_rawdata_asset]
is_enabled = true
storage_folder_name = "caravan/red_rawdata_asset"
table_name = "red_rawdata_asset"
select_file_name_base = "red_rawdata_asset"
regex_pattern = 'YouTube_(.+)_M_(\d{8}|\d{6})'
regex_group_names = ["content_owner", "file_date_key"]
table_append_or_replace = "append"
[file.red_rawdata_video]
is_enabled = true
storage_folder_name = "caravan/red_rawdata_video"
select_file_name_base = "red_rawdata_video"
table_name = "red_rawdata_video"
table_append_or_replace = "append"
regex_pattern = 'YouTube_(.+)_M_(\d{8}|\d{6})'
regex_group_names = ["content_owner", "file_date_key"]
[file.red_music_rawdata_asset]
is_enabled = true
storage_folder_name = "caravan/red_music_rawdata_asset"
select_file_name_base = "red_music_rawdata_asset"
table_name = "red_music_rawdata_asset"
table_append_or_replace = "append"
regex_pattern = 'YouTube_(.+)_M_(\d{8}|\d{6})'
regex_group_names = ["content_owner", "file_date_key"]
[file.red_music_rawdata_video]
is_enabled = true
storage_folder_name = "caravan/red_music_rawdata_video"
select_file_name_base = "red_music_rawdata_video"
table_name = "red_music_rawdata_video"
table_append_or_replace = "append"
regex_pattern = 'YouTube_(.+)_M_(\d{8}|\d{6})'
regex_group_names = ["content_owner", "file_date_key"]
[file.monthly_shorts_non_music_ads_video_summary]
is_enabled = true
storage_folder_name = "caravan/monthly_shorts_non_music_ads_video_summary"
select_file_name_base = "monthly_shorts_non_music_ads_video_summary"
table_name = "monthly_shorts_non_music_ads_video_summary"
table_append_or_replace = "append"
regex_pattern = 'YouTube_(.+)_M_(\d{8}|\d{6})'
regex_group_names = ["content_owner", "file_date_key"]
[file.monthly_shorts_non_music_subscription_video_summary]
is_enabled = true
storage_folder_name = "caravan/monthly_shorts_non_music_subscription_video_summary"
select_file_name_base = "monthly_shorts_non_music_subscription_video_summary"
table_name = "monthly_shorts_non_music_subscription_video_summary"
table_append_or_replace = "append"
regex_pattern = 'YouTube_(.+)_M_(\d{8}|\d{6})'
regex_group_names = ["content_owner", "file_date_key"]
[file.video_report]
is_enabled = true
storage_folder_name = "caravan/video_report"
select_file_name_base = "video_report"
table_name = "video_report"
table_append_or_replace = "replace"
regex_pattern = 'video_report_(.+)_V_(.+)\.csv'
regex_group_names = ["content_owner", "file_version"]