background material titles fixes

hasadna · Sep 28, 2023 · 567d542 · 567d542
1 parent 020fc42
commit 567d542
Show file tree

Hide file tree

Showing 3 changed files with 8 additions and 10 deletions.
diff --git a/airflow/docker-compose.yaml b/airflow/docker-compose.yaml
@@ -3,7 +3,7 @@ version: '2.4'
 services:
 
   db:
-    image: postgres:15
+    image: postgres:10
     restart: unless-stopped
     ports:
       - "5432:5432"

diff --git a/airflow/knesset_data_pipelines/committees/background_material_titles.py b/airflow/knesset_data_pipelines/committees/background_material_titles.py
@@ -57,11 +57,8 @@ def iterate_new_titles():
                 committees_kns_documentcommitteesession doc
                 join committees_kns_committeesession sess
                     on sess."CommitteeSessionID" = doc."CommitteeSessionID"
-                left join committees_document_background_material_titles title
-                    on title."DocumentCommitteeSessionID" = doc."DocumentCommitteeSessionID"
             where
-                title.title is null
-                and doc."GroupTypeID" = 87
+                doc."GroupTypeID" = 87
                 and doc."FilePath" is not null
                 and sess."StartDate" > '2020-01-01'
             group by sess."CommitteeSessionID", sess."CommitteeID"
@@ -79,7 +76,6 @@ def iterate_new_titles():
                     'FilePath': str(title['FilePath']),
                     'title': str(title['title']),
                 }
-            break
 
 
 def get_titles(committee, row):
@@ -104,6 +100,8 @@ def get_titles(committee, row):
                 if f'{row["FilePath"]}--{row["title"]}' not in yielded_rows:
                     yield row
                     yielded_rows.add(f'{row["FilePath"]}--{row["title"]}')
+        if len(yielded_rows) > 0:
+            print(f'got {len(yielded_rows)} titles')
 
 
 def main():

diff --git a/airflow/knesset_data_pipelines/committees/common.py b/airflow/knesset_data_pipelines/committees/common.py
@@ -23,10 +23,10 @@ def get_committees_tree():
                     'category_desc': row.category_desc,
                 } for row in conn.execute('''
                     select
-                        "CommitteeID" committee_id,
-                        "ParentCommitteeID" parent_committee_id,
-                        "Name" name,
-                        "CategoryDesc" category_desc
+                        "CommitteeID" as committee_id,
+                        "ParentCommitteeID" as parent_committee_id,
+                        "Name" as name,
+                        "CategoryDesc" as category_desc
                     from committees_kns_committee
                 ''')
             }