Skip to content

Commit

Permalink
fix: remove unrequested columns (#147)
Browse files Browse the repository at this point in the history 
* remove unrequested ads columns

* fix adset schemas

* add missing custom audiences

* remove duplicate schema entries

* refactor campaign schema parsing

* refactor creative schema parsing

* fix locales type

* update custom audiences
  • Loading branch information
@pnadolny13
pnadolny13 authored Jan 24, 2024
1 parent fee4d79 commit dc35211
Show file tree
Hide file tree
Showing 5 changed files with 175 additions and 293 deletions.
122 changes: 1 addition & 121 deletions tap_facebook/streams/ads.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,6 @@
from singer_sdk.streams.core import REPLICATION_INCREMENTAL
from singer_sdk.typing import (
ArrayType,
DateTimeType,
IntegerType,
ObjectType,
PropertiesList,
Expand Down Expand Up @@ -50,7 +49,7 @@ class AdsStream(IncrementalFacebookStream):
"bid_amount",
]

columns_remaining = ["adlabels", "recommendations"] # noqa: RUF012
columns_remaining = ["adlabels"] # noqa: RUF012

name = "ads"
filter_entity = "ad"
Expand All @@ -66,17 +65,6 @@ class AdsStream(IncrementalFacebookStream):
Property("account_id", StringType),
Property("campaign_id", StringType),
Property("adset_id", StringType),
Property(
"adlabels",
ArrayType(
ObjectType(
Property("id", StringType),
Property("created_time", DateTimeType),
Property("name", StringType),
Property("updated_time", DateTimeType),
),
),
),
Property("bid_amount", IntegerType),
Property(
"bid_info",
Expand Down Expand Up @@ -167,115 +155,7 @@ class AdsStream(IncrementalFacebookStream):
),
),
),
Property("placement_specific_facebook_unsafe_substances", StringType),
Property("placement_specific_instagram_unsafe_substances", StringType),
Property("global_unsafe_substances", StringType),
Property("placement_specific_instagram_personal_attributes", StringType),
Property("global_personal_attributes", StringType),
Property("placement_specific_facebook_personal_attributes", StringType),
Property("placement_specific_instagram_nonexistent_functionality", StringType),
Property("global_nonexistent_functionality", StringType),
Property("placement_specific_facebook_nonexistent_functionality", StringType),
Property("placement_specific_facebook_advertising_policies", StringType),
Property("global_advertising_policies", StringType),
Property("global_spyware_or_malware", StringType),
Property("placement_specific_instagram_spyware_or_malware", StringType),
Property("placement_specific_facebook_spyware_or_malware", StringType),
Property("placement_specific_instagram_unrealistic_outcomes", StringType),
Property("global_unrealistic_outcomes", StringType),
Property("placement_specific_facebook_unrealistic_outcomes", StringType),
Property("placement_specific_facebook_brand_usage_in_ads", StringType),
Property("global_brand_usage_in_ads", StringType),
Property("global_personal_health_and_appearance", StringType),
Property(
"placement_specific_facebook_personal_health_and_appearance",
StringType,
),
Property(
"placement_specific_instagram_personal_health_and_appearance",
StringType,
),
Property(
"placement_specific_instagram_illegal_products_or_services",
StringType,
),
Property("global_illegal_products_or_services", StringType),
Property(
"placement_specific_facebook_illegal_products_or_services",
StringType,
),
Property("global_non_functional_landing_page", StringType),
Property("placement_specific_facebook_non_functional_landing_page", StringType),
Property(
"placement_specific_instagram_non_functional_landing_page",
StringType,
),
Property(
"placement_specific_instagram_commercial_exploitation_of_crises_and_controversial_events",
StringType,
),
Property(
"placement_specific_facebook_commercial_exploitation_of_crises_and_controversial_events",
StringType,
),
Property(
"global_commercial_exploitation_of_crises_and_controversial_events",
StringType,
),
Property("global_discriminatory_practices", StringType),
Property("placement_specific_facebook_discriminatory_practices", StringType),
Property("global_circumventing_systems", StringType),
Property("placement_specific_facebook_circumventing_systems", StringType),
Property("placement_specific_instagram_circumventing_systems", StringType),
Property("placement_specific_facebook_adult_content", StringType),
Property("placement_specific_facebook_sensational_content", StringType),
Property("global_adult_content", StringType),
Property("global_sensational_content", StringType),
Property("placement_specific_instagram_adult_content", StringType),
Property("placement_specific_instagram_brand_usage_in_ads", StringType),
Property("placement_specific_instagram_sensational_content", StringType),
Property(
"placement_specific_facebook_ads_about_social_issues_elections_or_politics",
StringType,
),
Property(
"placement_specific_instagram_ads_about_social_issues_elections_or_politics",
StringType,
),
Property("global_ads_about_social_issues_elections_or_politics", StringType),
Property("configured_status", StringType),
Property("conversion_domain", StringType),
Property(
"conversion_specs",
ArrayType(
ObjectType(
Property("action.type", ArrayType(StringType)),
Property("conversion_id", ArrayType(StringType)),
),
),
),
Property("placement_specific_instagram_advertising_policies", StringType),
Property("recommendation_data", ArrayType(Property("items", StringType))),
Property("application", ArrayType(Property("items", StringType))),
Property("dataset", ArrayType(Property("items", StringType))),
Property("event", ArrayType(Property("items", StringType))),
Property("event_creator", ArrayType(Property("items", StringType))),
Property("event_type", ArrayType(Property("items", StringType))),
Property("fb_pixel", ArrayType(Property("items", StringType))),
Property("fb_pixel_event", ArrayType(Property("items", StringType))),
Property("leadgen", ArrayType(Property("items", StringType))),
Property("object", ArrayType(Property("items", StringType))),
Property("object_domain", ArrayType(Property("items", StringType))),
Property("offer", ArrayType(Property("items", StringType))),
Property("offer_creator", ArrayType(Property("items", StringType))),
Property("offsite_pixel", ArrayType(Property("items", StringType))),
Property("page_parent", ArrayType(Property("items", StringType))),
Property("post_object", ArrayType(Property("items", StringType))),
Property("post_object_wall", ArrayType(Property("items", StringType))),
Property("question", ArrayType(Property("items", StringType))),
Property("question_creator", ArrayType(Property("items", StringType))),
Property("response", ArrayType(Property("items", StringType))),
Property("subtype", ArrayType(Property("items", StringType))),
).to_dict()

tap_stream_id = "ads"
169 changes: 90 additions & 79 deletions tap_facebook/streams/adsets.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -126,6 +126,15 @@ class AdsetsStream(IncrementalFacebookStream):
Property("application_id", StringType),
Property("product_set_id", StringType),
Property("offer_id", StringType),
Property("custom_conversion_id", IntegerType),
Property("custom_event_str", StringType),
Property("event_id", IntegerType),
Property("offline_conversion_data_set_id", IntegerType),
Property("pixel_aggregation_rule", StringType),
Property("place_page_set_id", IntegerType),
Property("product_catalog_id", IntegerType),
Property("retention_days", StringType),
Property("application_type", StringType),
),
),
Property("id", StringType),
Expand Down Expand Up @@ -186,30 +195,22 @@ class AdsetsStream(IncrementalFacebookStream):
Property("instagram_actor_id", StringType),
Property("review_feedback", StringType),
Property("rf_prediction_id", StringType),
Property("promoted_object_application_id", IntegerType),
Property("promoted_object_custom_conversion_id", IntegerType),
Property("promoted_object_custom_event_str", StringType),
Property("promoted_object_custom_event_type", StringType),
Property("promoted_object_event_id", IntegerType),
Property("promoted_object_object_store_url", StringType),
Property("promoted_object_offer_id", IntegerType),
Property("promoted_object_offline_conversion_data_set_id", IntegerType),
Property("promoted_object_page_id", IntegerType),
Property("promoted_object_pixel_aggregation_rule", StringType),
Property("promoted_object_pixel_id", IntegerType),
Property("promoted_object_pixel_rule", StringType),
Property("promoted_object_place_page_set_id", IntegerType),
Property("promoted_object_product_catalog_id", IntegerType),
Property("promoted_object_product_set_id", IntegerType),
Property("promoted_object_retention_days", StringType),
Property("promoted_object_application_type", StringType),
Property("bid_amount", IntegerType),
Property("bid_strategy", StringType),
Property(
"targeting",
ObjectType(
Property("age_max", IntegerType),
Property("age_min", IntegerType),
Property(
"custom_audiences",
ArrayType(
ObjectType(
Property("id", StringType),
Property("name", StringType),
),
),
),
Property(
"excluded_custom_audiences",
ArrayType(
Expand All @@ -232,70 +233,80 @@ class AdsetsStream(IncrementalFacebookStream):
Property("facebook_positions", ArrayType(StringType)),
Property("instagram_positions", ArrayType(StringType)),
Property("device_platforms", ArrayType(StringType)),
Property("app_install_state", StringType),
Property("audience_network_positions", ArrayType(StringType)),
Property("behaviors", ArrayType(ObjectType())),
Property("college_years", ArrayType(StringType)),
Property("connections", ArrayType(ObjectType())),
Property("education_majors", ArrayType(ObjectType())),
Property("education_schools", ArrayType(StringType)),
Property("education_statuses", ArrayType(StringType)),
Property(
"effective_audience_network_positions",
ArrayType(StringType),
),
Property("excluded_connections", ArrayType(ObjectType())),
Property(
"excluded_geo_locations",
ObjectType(
Property("countries", ArrayType(StringType)),
Property(
"country_groups",
ArrayType(StringType),
),
Property(
"custom_locations",
ArrayType(ObjectType()),
),
Property(
"electoral_district",
ArrayType(StringType),
),
Property("geo_markets", ArrayType(ObjectType())),
Property(
"location_types",
ArrayType(StringType),
),
Property("places", ArrayType(StringType)),
Property("regions", ArrayType(ObjectType())),
Property("cities", ArrayType(ObjectType())),
Property("zips", ArrayType(ObjectType())),
),
),
Property("excluded_publisher_categories", ArrayType(StringType)),
Property("excluded_publisher_list_ids", ArrayType(StringType)),
Property("excluded_user_device", ArrayType(StringType)),
Property("exclusions", ArrayType(ObjectType())),
Property("family_statuses", ArrayType(ObjectType())),
Property("flexible_spec", ArrayType(ObjectType())),
Property("friends_of_connections", ArrayType(ObjectType())),
Property(
"geo_locations",
ObjectType(
Property("cities", ArrayType(ObjectType())),
Property("country_groups", ArrayType(StringType)),
Property("custom_locations", ArrayType(StringType)),
Property("electoral_district", ArrayType(StringType)),
Property("geo_markets", ArrayType(ObjectType())),
Property("places", ArrayType(StringType)),
Property("regions", ArrayType(ObjectType())),
Property("zips", ArrayType(ObjectType())),
),
),
Property("income", ArrayType(ObjectType())),
Property("industries", ArrayType(ObjectType())),
Property("interests", ArrayType(StringType)),
Property("life_events", ArrayType(ObjectType())),
Property("locales", ArrayType(IntegerType)),
Property("relationship_statuses", ArrayType(StringType)),
Property("user_adclusters", ArrayType(ObjectType())),
Property("user_device", ArrayType(StringType)),
Property("user_os", ArrayType(StringType)),
Property("wireless_carrier", ArrayType(StringType)),
Property("work_employers", ArrayType(ObjectType())),
Property("work_positions", ArrayType(ObjectType())),
),
),
Property("targeting_app_install_state", StringType),
Property("targeting_audience_network_positions", ArrayType(StringType)),
Property("targeting_behaviors", ArrayType(StringType)),
Property("targeting_college_years", ArrayType(StringType)),
Property("targeting_connections", ArrayType(StringType)),
Property("targeting_education_majors", ArrayType(StringType)),
Property("targeting_education_schools", ArrayType(StringType)),
Property("targeting_education_statuses", ArrayType(StringType)),
Property(
"targeting_effective_audience_network_positions",
ArrayType(StringType),
),
Property("targeting_excluded_connections", ArrayType(StringType)),
Property("targeting_excluded_geo_locations_countries", ArrayType(StringType)),
Property(
"targeting_excluded_geo_locations_country_groups",
ArrayType(StringType),
),
Property(
"targeting_excluded_geo_locations_custom_locations",
ArrayType(StringType),
),
Property(
"targeting_excluded_geo_locations_electoral_district",
ArrayType(StringType),
),
Property("targeting_excluded_geo_locations_geo_markets", ArrayType(StringType)),
Property(
"targeting_excluded_geo_locations_location_types",
ArrayType(StringType),
),
Property("targeting_excluded_geo_locations_places", ArrayType(StringType)),
Property("targeting_excluded_geo_locations_regions", ArrayType(StringType)),
Property("targeting_excluded_geo_locations_cities", ArrayType(StringType)),
Property("targeting_excluded_geo_locations_zips", ArrayType(StringType)),
Property("targeting_excluded_publisher_categories", ArrayType(StringType)),
Property("targeting_excluded_publisher_list_ids", ArrayType(StringType)),
Property("targeting_excluded_user_device", ArrayType(StringType)),
Property("targeting_exclusions", ArrayType(StringType)),
Property("targeting_family_statuses", ArrayType(StringType)),
Property("targeting_flexible_spec", ArrayType(StringType)),
Property("targeting_friends_of_connections", ArrayType(StringType)),
Property("targeting_geo_locations_cities", ArrayType(StringType)),
Property("targeting_geo_locations_country_groups", ArrayType(StringType)),
Property("targeting_geo_locations_custom_locations", ArrayType(StringType)),
Property("targeting_geo_locations_electoral_district", ArrayType(StringType)),
Property("targeting_geo_locations_geo_markets", ArrayType(StringType)),
Property("targeting_geo_locations_places", ArrayType(StringType)),
Property("targeting_geo_locations_regions", ArrayType(StringType)),
Property("targeting_geo_locations_zips", ArrayType(StringType)),
Property("targeting_income", ArrayType(StringType)),
Property("targeting_industries", ArrayType(StringType)),
Property("targeting_interests", ArrayType(StringType)),
Property("targeting_life_events", ArrayType(StringType)),
Property("targeting_locales", ArrayType(StringType)),
Property("targeting_relationship_statuses", ArrayType(StringType)),
Property("targeting_user_adclusters", ArrayType(StringType)),
Property("targeting_user_device", ArrayType(StringType)),
Property("targeting_user_os", ArrayType(StringType)),
Property("targeting_wireless_carrier", ArrayType(StringType)),
Property("targeting_work_employers", ArrayType(StringType)),
Property("targeting_work_positions", ArrayType(StringType)),
Property("lifetime_min_spend_target", StringType),
Property("lifetime_spend_cap", StringType),
).to_dict()
Expand Down
Loading

0 comments on commit dc35211

Please sign in to comment.