updated api script

Author: paul-stork

.gitignore

@@ -1 +1,2 @@
 /.env
+*.csv

coinAPI-pull.py

@@ -12,12 +12,13 @@
 # Finally, the dataframe that has each crytpo coin value in 30 minute increments is written to a PostgreSQL db table.
 
 from dotenv import load_dotenv
+import numpy as np
 import os
 import requests
 import json
 import pandas as pd
 from datetime import datetime, timedelta
-from sqlalchemy import exc, create_engine
+from sqlalchemy import exc, create_engine, text
 import time
 
 load_dotenv()
@@ -50,22 +51,23 @@
         if "status" not in daily_text:
             temp_df = pd.read_json(daily_text)
             temp_df['exchange_id'] = exchange_id
-            daily_df = pd.concat([daily_df, temp_df])
+            daily_df = pd.concat([daily_df, temp_df], ignore_index=True)
 
             time.sleep(3)
         else: 
             time.sleep(15)
     except requests.HTTPError as e:
         print(f'There was an error: {e}')
-
+        
+daily_df[['time_close', 'time_open', 'time_period_end', 'time_period_start']] = daily_df[['time_close', 'time_open', 'time_period_end', 'time_period_start']].apply(pd.to_datetime, utc=True)
+        
 
 daily_df = daily_df[['exchange_id','rate_close','rate_high','rate_low','rate_open','time_close','time_open','time_period_end','time_period_start']]
-
-# print(daily_df)
+daily_df = daily_df.drop_duplicates().sort_values(by='time_period_start', ignore_index=True)
 
 try:    
-    daily_df.to_sql(name = 'daily_crypto_data', con=conn, if_exists='append', index=False)
-
+    daily_df.to_sql(name = 'daily_crypto_prod', schema='crypto_data', con=conn, if_exists='append', index=False)
+        
     print('data loaded to table')
 except exc.SQLAlchemyError as e:
     print(f'There was an error: {e}')

requirements.txt

@@ -1,20 +1,47 @@
-certifi==2025.1.31
-charset-normalizer==3.4.1
+anaconda-anon-usage @ file:///croot/anaconda-anon-usage_1710965072196/work
+archspec @ file:///croot/archspec_1709217642129/work
+boltons @ file:///work/perseverance-python-buildout/croot/boltons_1698851177130/work
+Brotli @ file:///croot/brotli-split_1714483155106/work
+certifi @ file:///croot/certifi_1725551672989/work/certifi
+cffi @ file:///croot/cffi_1726856441404/work
+charset-normalizer @ file:///croot/charset-normalizer_1721748349566/work
+conda @ file:///croot/conda_1729193882500/work
+conda-content-trust @ file:///croot/conda-content-trust_1714483159009/work
+conda-libmamba-solver @ file:///croot/conda-libmamba-solver_1727775630457/work/src
+conda-package-handling @ file:///croot/conda-package-handling_1718138267740/work
+conda_package_streaming @ file:///croot/conda-package-streaming_1718136078615/work
+cryptography @ file:///croot/cryptography_1724940562255/work
+distro @ file:///croot/distro_1714488253808/work
+frozendict @ file:///croot/frozendict_1713194832637/work
 greenlet==3.1.1
-idna==3.10
+idna @ file:///croot/idna_1714398848350/work
+jsonpatch @ file:///croot/jsonpatch_1714483231291/work
+jsonpointer==2.1
+libmambapy @ file:///croot/mamba-split_1725563367192/work/libmambapy
+menuinst @ file:///croot/menuinst_1723567589013/work
 numpy==2.2.2
+packaging @ file:///croot/packaging_1720101850331/work
 pandas==2.2.3
+platformdirs @ file:///work/perseverance-python-buildout/croot/platformdirs_1701732573265/work
+pluggy @ file:///work/perseverance-python-buildout/croot/pluggy_1698805497733/work
 psycopg==3.2.4
 psycopg-binary==3.2.4
-psycopg-pool==3.2.4
+pycosat @ file:///croot/pycosat_1714510623388/work
+pycparser @ file:///tmp/build/80754af9/pycparser_1636541352034/work
+PySocks @ file:///work/perseverance-python-buildout/croot/pysocks_1698845478203/work
 python-dateutil==2.9.0.post0
 python-dotenv==1.0.1
 pytz==2025.1
-requests==2.32.3
-setuptools==75.8.0
+requests @ file:///croot/requests_1721410876868/work
+ruamel.yaml @ file:///croot/ruamel.yaml_1727980156079/work
+ruamel.yaml.clib @ file:///croot/ruamel.yaml.clib_1727769819935/work
+setuptools==75.1.0
 six==1.17.0
-SQLAlchemy==2.0.37
+SQLAlchemy==2.0.38
+tqdm @ file:///croot/tqdm_1724853939799/work
+truststore @ file:///work/perseverance-python-buildout/croot/truststore_1701735771625/work
 typing_extensions==4.12.2
 tzdata==2025.1
-urllib3==2.3.0
-wheel==0.45.1
+urllib3 @ file:///croot/urllib3_1727769808118/work
+wheel==0.44.0
+zstandard @ file:///croot/zstandard_1728569189425/work