Merge pull request #3438 from esdc-esac-esa-int/ESA_euclid-EUCLIDSWRQ-215-get_datalinks_metadata

Author: bsipocz

CHANGES.rst

@@ -9,6 +9,12 @@ New Tools and Services
 API changes
 -----------
 
+esa.euclid
+^^^^^^^^^^
+
+- Method ``get_datalinks`` has a new argument, ``extra_options``, to customize the results to be got
+  from the server. Specifically, passing ``'METADATA'`` to this argument will retrieve the extra fields
+  ``datalabs_path``, ``file_name`` and ``hdu_index``. [#3438]
 
 
 Service fixes and enhancements

astroquery/esa/euclid/core.py

@@ -1418,8 +1418,11 @@ def get_spectrum(self, *, source_id, schema='sedm', retrieval_type="ALL", output
 
         return files
 
-    def get_datalinks(self, ids, *, linking_parameter='SOURCE_ID', verbose=False):
-        """Gets datalinks associated to the provided identifiers
+    def get_datalinks(self, ids, *, linking_parameter='SOURCE_ID', extra_options=None, verbose=False):
+        """
+        Description
+        -----------
+        Gets datalinks associated to the provided identifiers.
         TAP+ only
 
         Parameters
@@ -1428,6 +1431,9 @@ def get_datalinks(self, ids, *, linking_parameter='SOURCE_ID', verbose=False):
             list of identifiers
         linking_parameter : str, optional, default SOURCE_ID, valid values: SOURCE_ID
             By default, all the identifiers are considered as source_id
+        extra_options : str, optional, default None, valid values: METADATA
+            To let customize the server behaviour, if present.
+            If provided with value METADATA, the extra fields datalabs_path, file_name & hdu_index will be retrieved.
         verbose : bool, optional, default 'False'
             flag to display information about the process
 
@@ -1437,7 +1443,10 @@ def get_datalinks(self, ids, *, linking_parameter='SOURCE_ID', verbose=False):
 
         """
 
-        return self.__eucliddata.get_datalinks(ids=ids, linking_parameter=linking_parameter, verbose=verbose)
+        return self.__eucliddata.get_datalinks(ids=ids,
+                                               linking_parameter=linking_parameter,
+                                               extra_options=extra_options,
+                                               verbose=verbose)
 
     def get_scientific_product_list(self, *, observation_id=None, tile_index=None, category=None, group=None,
                                     product_type=None, dataset_release='REGREPROC1_R2', verbose=False):

astroquery/esa/euclid/tests/test_euclidtap.py

@@ -1220,19 +1220,36 @@ def test_logout(mock_logout):
 
 
 def test_get_datalinks(monkeypatch):
-    def get_datalinks_monkeypatched(self, ids, linking_parameter, verbose):
+    def get_datalinks_monkeypatched(self, ids, linking_parameter, extra_options, verbose):
         return Table()
 
     # `EuclidClass` is a subclass of `TapPlus`, but it does not inherit
     # `get_datalinks()`, it replaces it with a call to the `get_datalinks()`
-    # of its `__gaiadata`.
+    # of its `__eucliddata`.
     monkeypatch.setattr(TapPlus, "get_datalinks", get_datalinks_monkeypatched)
     euclid = EuclidClass(show_server_messages=False)
 
     result = euclid.get_datalinks(ids=[12345678], verbose=True)
     assert isinstance(result, Table)
 
 
+def test_get_datalinks_with_metadata(monkeypatch):
+    def get_datalinks_monkeypatched(self, ids, linking_parameter, extra_options, verbose):
+        table = TapTableMeta()
+        table.name = extra_options
+        return table
+
+    # `EuclidClass` is a subclass of `TapPlus`, but it does not inherit
+    # `get_datalinks()`, it replaces it with a call to the `get_datalinks()`
+    # of its `__eucliddata`.
+    monkeypatch.setattr(TapPlus, "get_datalinks", get_datalinks_monkeypatched)
+    euclid = EuclidClass(show_server_messages=False)
+
+    result = euclid.get_datalinks(ids=[12345678], extra_options='METADATA')
+    assert isinstance(result, TapTableMeta)
+    assert result.name == "METADATA"
+
+
 @pytest.mark.parametrize("background", [False, True])
 def test_cross_match_basic(monkeypatch, background, cross_match_basic_kwargs, mock_querier_async):
     def load_table_monkeypatched(self, table, verbose):

astroquery/utils/tap/core.py

@@ -1247,20 +1247,23 @@ def is_valid_user(self, *, user_id=None, verbose=False):
             print(f"USER response = {user}")
         return user.startswith(f"{user_id}:") and user.count("\\n") == 0
 
-    def get_datalinks(self, ids, *, linking_parameter=None, verbose=False):
+    def get_datalinks(self, ids, *, linking_parameter=None, extra_options=None, verbose=False):
         """Gets datalinks associated to the provided identifiers
 
         Parameters
         ----------
         ids : str list, mandatory
-            list of identifiers
+            List of identifiers
         linking_parameter : str, optional, default SOURCE_ID, valid values: SOURCE_ID, TRANSIT_ID, IMAGE_ID
             By default, all the identifiers are considered as source_id
             SOURCE_ID: the identifiers are considered as source_id
             TRANSIT_ID: the identifiers are considered as transit_id
             IMAGE_ID: the identifiers are considered as sif_observation_id
+        extra_options : str, optional, default None, valid values: METADATA
+            If present, an extra parameter OPTIONS will be added to the call, to be interpreted by the TAP service
+            METADATA: to retrieve extra metadata columns (currently supported by the Euclid archive)
         verbose : bool, optional, default 'False'
-            flag to display information about the process
+            Flag to display information about the process
 
         Returns
         -------
@@ -1282,15 +1285,17 @@ def get_datalinks(self, ids, *, linking_parameter=None, verbose=False):
         if linking_parameter is not None:
             ids_arg = f'{ids_arg}&LINKING_PARAMETER={linking_parameter}'
 
+        if extra_options is not None:
+            ids_arg = f'{ids_arg}&OPTIONS={extra_options}'
+
         if verbose:
-            print(f"Datalink request: {ids_arg}")
-        connHandler = self.__getconnhandler()
-        response = connHandler.execute_datalinkpost(subcontext="links",
-                                                    data=ids_arg,
-                                                    verbose=verbose)
+            print(f"Datalink request: ID={ids_arg}")
+
+        conn_handler = self.__getconnhandler()
+        response = conn_handler.execute_datalinkpost(subcontext="links", data=ids_arg, verbose=verbose)
         if verbose:
             print(response.status, response.reason)
-        connHandler.check_launch_response_status(response, verbose, 200)
+        conn_handler.check_launch_response_status(response, verbose, 200)
         if verbose:
             print("Done.")
         results = utils.read_http_response(response, "votable", use_names_over_ids=self.use_names_over_ids)

astroquery/utils/tap/tests/test_tap.py

@@ -635,6 +635,8 @@ def test_datalink():
     responseResultsJob.set_data(method='GET', body=TEST_DATA["job_1.vot"])
     req = "links?ID=1,2"
     conn_handler.set_response(req, responseResultsJob)
+    req = "links?ID=1&OPTIONS=METADATA"
+    conn_handler.set_response(req, responseResultsJob)
 
     # error
     responseResultsJob.set_status_code(500)
@@ -651,6 +653,8 @@ def test_datalink():
     assert len(results) == 3
     results = tap.get_datalinks(['1', '2'])
     assert len(results) == 3
+    results = tap.get_datalinks(1, extra_options='METADATA', verbose=True)
+    assert len(results) == 3
 
 
 def test_get_new_column_values_for_update():

docs/esa/euclid/euclid.rst

@@ -1216,6 +1216,20 @@ To find out the resources associated with a given source:
   sedm 2707008224650763513         SOURCE_ID https://eas.esac.esa.int/sas-dd/data?ID=sedm+2707008224650763513&RETRIEVAL_TYPE=SPECTRA_BGS                               #this Spectra Blue Source                          --
 
 
+This method also provides a way to get the extra columns datalabs_path, file_name & hdu_index in the call:
+
+.. Skipping authentication requiring examples
+.. doctest-skip::
+
+  >>> from astroquery.esa.euclid import Euclid
+  >>> Euclid.login()
+  >>> result = Euclid.get_datalinks(ids=2707008224650763513, extra_options='METADATA')
+  >>> print(result)
+             ID            linking_parameter                                       access_url                                            ...                           file_name                           hdu_index
+                                                                                                                                         ...
+  ------------------------ ----------------- ------------------------------------------------------------------------------------------- ... ------------------------------------------------------------- ---------
+  sedm 2707008224650763513         SOURCE_ID https://eas.esac.esa.int/sas-dd/data?ID=sedm+2707008224650763513&RETRIEVAL_TYPE=SPECTRA_RGS ... EUC_SIR_W-COMBSPEC_102158586_2024-11-05T16:05:44.880543Z.fits      1602
+  sedm 2707008224650763513         SOURCE_ID https://eas.esac.esa.int/sas-dd/data?ID=sedm+2707008224650763513&RETRIEVAL_TYPE=SPECTRA_BGS ... EUC_SIR_W-COMBSPEC_102158586_2024-11-05T16:05:44.880543Z.fits      1602
 
 
 The query below retrieves a random sample of Euclid sources having spectra.