Skip to content

Commit 8192801

Browse files
bsipoczbsipocz
authored
Merge pull request #3193 from snbianco/ASB-30325-mast-uris
Accept MAST URIs as input to get_cloud_uris()
2 parents d619223 + 311197b commit 8192801

10 files changed

+234
-48
lines changed

‎CHANGES.rst

+14
Original file line numberDiff line numberDiff line change
@@ -4,9 +4,16 @@
44
New Tools and Services
55
----------------------
66

7+
78
API changes
89
-----------
910

11+
mast
12+
^^^^
13+
14+
- Handle a MAST URI string as input for ``Observations.get_cloud_uri`` and a list of MAST URIs as input for
15+
``Observations.get_cloud_uris``. [#3193]
16+
1017
simbad
1118
^^^^^^
1219

@@ -35,6 +42,13 @@ ipac.nexsci.nasa_exoplanet_archive
3542

3643
- Fixed InvalidTableError for DI_STARS_EXEP and TD tables. [#3189]
3744

45+
mast
46+
^^^^
47+
48+
- Bugfix where users are unnecessarily warned about a query limit while fetching products in ``MastMissions.get_product_list``. [#3193]
49+
50+
- Bugfix where ``Observations.get_cloud_uri`` and ``Observations.get_cloud_uris`` fail if the MAST relative path is not found. [#3193]
51+
3852
simbad
3953
^^^^^^
4054

‎astroquery/mast/cloud.py

+10-11
Original file line numberDiff line numberDiff line change
@@ -52,7 +52,8 @@ def __init__(self, provider="AWS", profile=None, verbose=False):
5252
import boto3
5353
import botocore
5454

55-
self.supported_missions = ["mast:hst/product", "mast:tess/product", "mast:kepler", "mast:galex", "mast:ps1"]
55+
self.supported_missions = ["mast:hst/product", "mast:tess/product", "mast:kepler", "mast:galex", "mast:ps1",
56+
"mast:jwst/product"]
5657

5758
self.boto3 = boto3
5859
self.botocore = botocore
@@ -77,11 +78,7 @@ def is_supported(self, data_product):
7778
response : bool
7879
Is the product from a supported mission.
7980
"""
80-
81-
for mission in self.supported_missions:
82-
if data_product['dataURI'].lower().startswith(mission):
83-
return True
84-
return False
81+
return any(data_product['dataURI'].lower().startswith(mission) for mission in self.supported_missions)
8582

8683
def get_cloud_uri(self, data_product, include_bucket=True, full_url=False):
8784
"""
@@ -92,7 +89,7 @@ def get_cloud_uri(self, data_product, include_bucket=True, full_url=False):
9289
9390
Parameters
9491
----------
95-
data_product : `~astropy.table.Row`
92+
data_product : `~astropy.table.Row`, str
9693
Product to be converted into cloud data uri.
9794
include_bucket : bool
9895
Default True. When false returns the path of the file relative to the
@@ -108,6 +105,8 @@ def get_cloud_uri(self, data_product, include_bucket=True, full_url=False):
108105
Cloud URI generated from the data product. If the product cannot be
109106
found in the cloud, None is returned.
110107
"""
108+
# If data_product is a string, convert to a list
109+
data_product = [data_product] if isinstance(data_product, str) else data_product
111110

112111
uri_list = self.get_cloud_uri_list(data_product, include_bucket=include_bucket, full_url=full_url)
113112

@@ -124,8 +123,8 @@ def get_cloud_uri_list(self, data_products, include_bucket=True, full_url=False)
124123
125124
Parameters
126125
----------
127-
data_products : `~astropy.table.Table`
128-
Table containing products to be converted into cloud data uris.
126+
data_products : `~astropy.table.Table`, list
127+
Table containing products or list of MAST uris to be converted into cloud data uris.
129128
include_bucket : bool
130129
Default True. When false returns the path of the file relative to the
131130
top level cloud storage location.
@@ -141,8 +140,8 @@ def get_cloud_uri_list(self, data_products, include_bucket=True, full_url=False)
141140
if data_products includes products not found in the cloud.
142141
"""
143142
s3_client = self.boto3.client('s3', config=self.config)
144-
145-
paths = utils.mast_relative_path(data_products["dataURI"])
143+
data_uris = data_products if isinstance(data_products, list) else data_products['dataURI']
144+
paths = utils.mast_relative_path(data_uris)
146145
if isinstance(paths, str): # Handle the case where only one product was requested
147146
paths = [paths]
148147

‎astroquery/mast/missions.py

+5-4
Original file line numberDiff line numberDiff line change
@@ -99,14 +99,15 @@ def _parse_result(self, response, *, verbose=False): # Used by the async_to_syn
9999

100100
if self.service == self._search:
101101
results = self._service_api_connection._parse_result(response, verbose, data_key='results')
102+
103+
# Warn if maximum results are returned
104+
if len(results) >= self.limit:
105+
warnings.warn("Maximum results returned, may not include all sources within radius.",
106+
MaxResultsWarning)
102107
elif self.service == self._list_products:
103108
# Results from post_list_products endpoint need to be handled differently
104109
results = Table(response.json()['products'])
105110

106-
if len(results) >= self.limit:
107-
warnings.warn("Maximum results returned, may not include all sources within radius.",
108-
MaxResultsWarning)
109-
110111
return results
111112

112113
def _validate_criteria(self, **criteria):

‎astroquery/mast/observations.py

+15-8
Original file line numberDiff line numberDiff line change
@@ -854,9 +854,9 @@ def get_cloud_uris(self, data_products=None, *, include_bucket=True, full_url=Fa
854854
855855
Parameters
856856
----------
857-
data_products : `~astropy.table.Table`
858-
Table containing products to be converted into cloud data uris. If provided, this will supercede
859-
page_size, page, or any keyword arguments passed in as criteria.
857+
data_products : `~astropy.table.Table`, list
858+
Table containing products or list of MAST uris to be converted into cloud data uris.
859+
If provided, this will supercede page_size, page, or any keyword arguments passed in as criteria.
860860
include_bucket : bool
861861
Default True. When False, returns the path of the file relative to the
862862
top level cloud storage location.
@@ -920,16 +920,23 @@ def get_cloud_uris(self, data_products=None, *, include_bucket=True, full_url=Fa
920920
# Return list of associated data products
921921
data_products = self.get_product_list(obs)
922922

923-
# Filter product list
924-
data_products = self.filter_products(data_products, mrp_only=mrp_only, extension=extension, **filter_products)
923+
if isinstance(data_products, Table):
924+
# Filter product list
925+
data_products = self.filter_products(data_products, mrp_only=mrp_only, extension=extension,
926+
**filter_products)
927+
else: # data_products is a list of URIs
928+
# Warn if trying to supply filters
929+
if filter_products or extension or mrp_only:
930+
warnings.warn('Filtering is not supported when providing a list of MAST URIs. '
931+
'To apply filters, please provide query criteria or a table of data products '
932+
'as returned by `Observations.get_product_list`', InputWarning)
925933

926934
if not len(data_products):
927-
warnings.warn("No matching products to fetch associated cloud URIs.", NoResultsWarning)
935+
warnings.warn('No matching products to fetch associated cloud URIs.', NoResultsWarning)
928936
return
929937

930938
# Remove duplicate products
931939
data_products = utils.remove_duplicate_products(data_products, 'dataURI')
932-
933940
return self._cloud_connection.get_cloud_uri_list(data_products, include_bucket, full_url)
934941

935942
def get_cloud_uri(self, data_product, *, include_bucket=True, full_url=False):
@@ -941,7 +948,7 @@ def get_cloud_uri(self, data_product, *, include_bucket=True, full_url=False):
941948
942949
Parameters
943950
----------
944-
data_product : `~astropy.table.Row`
951+
data_product : `~astropy.table.Row`, str
945952
Product to be converted into cloud data uri.
946953
include_bucket : bool
947954
Default True. When false returns the path of the file relative to the

‎astroquery/mast/tests/data/README.rst

+12
Original file line numberDiff line numberDiff line change
@@ -36,3 +36,15 @@ To generate `~astroquery.mast.tests.data.mission_products.json`, use the followi
3636
>>> resp = utils._simple_request('https://mast.stsci.edu/search/hst/api/v0.1/list_products', {'dataset_ids': 'Z14Z0104T'})
3737
>>> with open('panstarrs_columns.json', 'w') as file:
3838
... json.dump(resp.json(), file, indent=4) # doctest: +SKIP
39+
40+
To generate `~astroquery.mast.tests.data.mast_relative_path.json`, use the following:
41+
42+
.. doctest-remote-data::
43+
44+
>>> import json
45+
>>> from astroquery.mast import utils
46+
...
47+
>>> resp = utils._simple_request('https://mast.stsci.edu/api/v0.1/path_lookup/',
48+
... {'uri': ['mast:HST/product/u9o40504m_c3m.fits', 'mast:HST/product/does_not_exist.fits']})
49+
>>> with open('mast_relative_path.json', 'w') as file:
50+
... json.dump(resp.json(), file, indent=4) # doctest: +SKIP

‎astroquery/mast/tests/data/mast_relative_path.json

+10
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,10 @@
1+
{
2+
"mast:HST/product/u9o40504m_c3m.fits": {
3+
"status_code": 200,
4+
"path": "/hst/public/u9o4/u9o40504m/u9o40504m_c3m.fits"
5+
},
6+
"mast:HST/product/does_not_exist.fits": {
7+
"status_code": 404,
8+
"path": null
9+
}
10+
}

‎astroquery/mast/tests/test_mast.py

+95-1
Original file line numberDiff line numberDiff line change
@@ -4,6 +4,7 @@
44
import os
55
import re
66
from shutil import copyfile
7+
from unittest.mock import patch
78

89
import pytest
910

@@ -16,7 +17,8 @@
1617

1718
from astroquery.mast.services import _json_to_table
1819
from astroquery.utils.mocks import MockResponse
19-
from astroquery.exceptions import InvalidQueryError, InputWarning, MaxResultsWarning
20+
from astroquery.exceptions import (InvalidQueryError, InputWarning, MaxResultsWarning, NoResultsWarning,
21+
RemoteServiceError)
2022

2123
from astroquery import mast
2224

@@ -48,6 +50,7 @@
4850
'Mast.HscMatches.Db.v3': 'matchid.json',
4951
'Mast.HscMatches.Db.v2': 'matchid.json',
5052
'Mast.HscSpectra.Db.All': 'spectra.json',
53+
'mast_relative_path': 'mast_relative_path.json',
5154
'panstarrs': 'panstarrs.json',
5255
'panstarrs_columns': 'panstarrs_columns.json',
5356
'tess_cutout': 'astrocut_107.27_-70.0_5x5.zip',
@@ -142,6 +145,8 @@ def request_mockreturn(url, params={}):
142145
filename = data_path(DATA_FILES["Mast.Name.Lookup"])
143146
elif 'panstarrs' in url:
144147
filename = data_path(DATA_FILES['panstarrs_columns'])
148+
elif 'path_lookup' in url:
149+
filename = data_path(DATA_FILES['mast_relative_path'])
145150
with open(filename, 'rb') as infile:
146151
content = infile.read()
147152
return MockResponse(content)
@@ -678,6 +683,95 @@ def test_observations_download_file(patch_post, tmpdir):
678683
assert result == ('COMPLETE', None, None)
679684

680685

686+
@patch('boto3.client')
687+
def test_observations_get_cloud_uri(mock_client, patch_post):
688+
pytest.importorskip("boto3")
689+
690+
mast_uri = 'mast:HST/product/u9o40504m_c3m.fits'
691+
expected = 's3://stpubdata/hst/public/u9o4/u9o40504m/u9o40504m_c3m.fits'
692+
693+
# Error without cloud connection
694+
with pytest.raises(RemoteServiceError):
695+
mast.Observations.get_cloud_uri('mast:HST/product/u9o40504m_c3m.fits')
696+
697+
# Enable access to public AWS S3 bucket
698+
mast.Observations.enable_cloud_dataset()
699+
700+
# Row input
701+
product = Table()
702+
product['dataURI'] = [mast_uri]
703+
uri = mast.Observations.get_cloud_uri(product[0])
704+
assert isinstance(uri, str)
705+
assert uri == expected
706+
707+
# String input
708+
uri = mast.Observations.get_cloud_uri(mast_uri)
709+
assert uri == expected
710+
711+
mast.Observations.disable_cloud_dataset()
712+
713+
714+
@patch('boto3.client')
715+
def test_observations_get_cloud_uris(mock_client, patch_post):
716+
pytest.importorskip("boto3")
717+
718+
mast_uri = 'mast:HST/product/u9o40504m_c3m.fits'
719+
expected = 's3://stpubdata/hst/public/u9o4/u9o40504m/u9o40504m_c3m.fits'
720+
721+
# Error without cloud connection
722+
with pytest.raises(RemoteServiceError):
723+
mast.Observations.get_cloud_uris(['mast:HST/product/u9o40504m_c3m.fits'])
724+
725+
# Enable access to public AWS S3 bucket
726+
mast.Observations.enable_cloud_dataset()
727+
728+
# Get the cloud URIs
729+
# Table input
730+
product = Table()
731+
product['dataURI'] = [mast_uri]
732+
uris = mast.Observations.get_cloud_uris([mast_uri])
733+
assert isinstance(uris, list)
734+
assert len(uris) == 1
735+
assert uris[0] == expected
736+
737+
# List input
738+
uris = mast.Observations.get_cloud_uris([mast_uri])
739+
assert isinstance(uris, list)
740+
assert len(uris) == 1
741+
assert uris[0] == expected
742+
743+
# Warn if attempting to filter with list input
744+
with pytest.warns(InputWarning, match='Filtering is not supported'):
745+
mast.Observations.get_cloud_uris([mast_uri],
746+
extension='png')
747+
748+
# Warn if not found
749+
with pytest.warns(NoResultsWarning, match='Failed to retrieve MAST relative path'):
750+
mast.Observations.get_cloud_uris(['mast:HST/product/does_not_exist.fits'])
751+
752+
753+
@patch('boto3.client')
754+
def test_observations_get_cloud_uris_query(mock_client, patch_post):
755+
pytest.importorskip("boto3")
756+
757+
# enable access to public AWS S3 bucket
758+
mast.Observations.enable_cloud_dataset()
759+
760+
# get uris with streamlined function
761+
uris = mast.Observations.get_cloud_uris(target_name=234295610,
762+
filter_products={'productSubGroupDescription': 'C3M'})
763+
assert isinstance(uris, list)
764+
765+
# check that InvalidQueryError is thrown if neither data_products or **criteria are defined
766+
with pytest.raises(InvalidQueryError):
767+
mast.Observations.get_cloud_uris(filter_products={'productSubGroupDescription': 'C3M'})
768+
769+
# warn if no data products match filters
770+
with pytest.warns(NoResultsWarning, match='No matching products'):
771+
mast.Observations.get_cloud_uris(target_name=234295610,
772+
filter_products={'productSubGroupDescription': 'LC'})
773+
774+
681775
######################
682776
# CatalogClass tests #
683777
######################

0 commit comments

Comments
 (0)