From fc5da488a2ec3c114b5497a3dee5947efc355339 Mon Sep 17 00:00:00 2001 From: Ian Nigel Evans Date: Tue, 1 Sep 2026 16:54:54 -0400 Subject: [PATCH] Close to final updates (1) Added in changes proposed by Karl and Matthias. This added several footnotes. The main change here is to change pointing_mode to array_pointing_mode, which makes sense because "pointing_mode" is very general and this particular attribute applies only to arrays. (2) Performed some grammatical/style updates to match the latest updates to the rest of the document. (3) Changed "Proposal" for a response table to "Preliminary Proposal". (4) Fixed formatting in a number of locations. (5) Updated Makefile date. --- Dataproducts-Summary-table.tex | 33 +++++++++------ HighEnergyObsCoreExt.tex | 40 +++++++++---------- Makefile | 2 +- UseCases.tex | 2 +- extendedAccessTonewtypesOfproducts.tex | 4 +- ...ndedAccessTonewtypesOfproductsAppendix.tex | 33 +++++++-------- 6 files changed, 62 insertions(+), 52 deletions(-) diff --git a/Dataproducts-Summary-table.tex b/Dataproducts-Summary-table.tex index cb1b4d6..14a2d76 100644 --- a/Dataproducts-Summary-table.tex +++ b/Dataproducts-Summary-table.tex @@ -3,39 +3,48 @@ %\newpage \subsubsection{Summary Table} -The proposed vocabulary entries are listed in Table~\ref{tab:dp_vocabulary} with their labels and parents identified. Two vocabularies are proposed for response functions and analysis products following the semantics procedure for VEPs (see \url{https://github.com/ivoa-std/VEPs}). +The proposed vocabulary entries are listed in Table~\ref{tab:dp_vocabulary} with their labels and parents identified. Two vocabularies are proposed for response functions and advanced data products following the semantics procedure for VEPs (see \url{https://github.com/ivoa-std/VEPs}). \begin{landscape} \begin{longtable}{p{0.1\linewidth}p{0.175\linewidth}p{0.475\linewidth}p{0.175\linewidth}} \sptablerule \textbf{Term} & \textbf{Label} & \textbf{Description} &\textbf{Parent}\cr \sptablerule -\noalign{\vspace{8pt}} +\noalign{\vspace{12pt}} \hline -\multicolumn{4}{|r|}{\bf Added to the IVOA Data Product Type Vocabulary} (\url{https://www.ivoa.net/rdf/product-type/}) \\ \hline -\noalign{\vspace{2pt}} +\multicolumn{4}{|r|}{{\bf Added to the IVOA Data Product Type Vocabular}y (\url{https://www.ivoa.net/rdf/product-type/})} \\ \hline +\noalign{\vspace{6pt}} {\bf hea-event-bundle} & Event Bundle & A compounded dataset containing containing an {\bf hea-event-list} and multiple files or other substructures that are products necessary to analyze the {\bf hea-event-list} & none \cr +\noalign{\vspace{2pt}} {\bf hea-event-list} & Event List & A dataset that records a collection of observed particle-detection events, such as incoming high-energy particles, where an event is typically characterized by a spatial position, a time, and a spectral value ({\em e.g.\/}, an energy, a channel, a pulse height) & \#temporally-resolved-dataset \cr -\noalign{\vspace{8pt}} +\noalign{\vspace{20pt}} \hline -\multicolumn{4}{|r|}{\bf Added to the IVOA Response Function Types Vocabulary} (\url{https://www.ivoa.net/rdf/response-type/}) \\ \hline -\noalign{\vspace{2pt}} +\multicolumn{4}{|r|}{{\bf Added to the IVOA Response Function Types Vocabulary} (\url{https://www.ivoa.net/rdf/response-type/})} \\ \hline +\noalign{\vspace{6pt}} {\bf response-function} & Response Function & A dataset that records the mapping from a physical quantity to an observable quantitiy. Narrower terms are preferred to indicate more precisely the type of {\bf response-function} & \cr +\noalign{\vspace{2pt}} {\bf rmf} &\raggedright Redistribution Matrix File & A dataset that records the probability density function mapping from energy space into detector pulse height (or position) space & \#response-function, \#pdf \cr +\noalign{\vspace{2pt}} {\bf aeff} & Effective Area & A dataset that records the ``effective area'' of a telescope and/or instrument. The effective area is the geometric area of the telescope and/or instrument reduced by efficiency factors such as reflectivity and vignetting, among other effects & \#response-function \cr +\noalign{\vspace{2pt}} {\bf arf} &\raggedright Ancillary Response File & A dataset that records the combined telescope/instrument effective area and detector quantum efficiency as a function of energy & \#response-function \cr +\noalign{\vspace{2pt}} {\bf bkgrate} & Background Rate & A dataset that models the rate of residual events that are not from the expected source type ({\em e.g.\/}, for gamma-ray instruments {\bf bkgrate} measures residual non-gamma-ray events coming from charged cosmic rays) & \#response-function \cr +\noalign{\vspace{2pt}} {\bf edisp} & Energy Dispersion & A dataset that records the probability density of detecting an event with an energy estimator (proxy) given the true energy of the event & \#response-function, \#pdf \cr +\noalign{\vspace{2pt}} {\bf psf} &\raggedright Point Spread Function & A dataset that records the probability density function of spatial/angular spreading of incident particles from a point source caused by the instrument (detector and/or mirror and/or analysis) & \#response-function, \#pdf \cr -\noalign{\vspace{8pt}} +\noalign{\vspace{20pt}} \hline -\multicolumn{4}{|r|}{\bf Considered for addition to an IVOA Advanced Data Product Vocabulary} \ \\ \hline -\noalign{\vspace{2pt}} +\multicolumn{4}{|r|}{{\bf Considered for addition to an IVOA Advanced Data Products Vocabulary}} \\ \hline +\noalign{\vspace{6pt}} {\bf draws} & Draws & A dataset that records statistical draws computed from a probability distribution or a sample population, for example Markov chain Monte Carlo (MCMC) draws used when computing the Bayesian marginal probability density function for a random variable, or the DeltaTS associated with a quantity from a frequentist analysis & none \cr +\noalign{\vspace{2pt}} {\bf pdf} &\raggedright Probability Density Function & A dataset that records the probability density function of a quantity, for example the Bayesian marginal probability density function for a random variable & none \cr +\noalign{\vspace{2pt}} {\bf region} & Region & A dataset that encodes (one or more) regions of parameter space, for example a spatial region or a region of phase space covered by a dataset. The set of dimensions represented by the region can be arbitrary & none \cr -%\sptablerule -\caption{IVOA Vocabulary Extension for High energy data products.} +% \sptablerule +\caption{IVOA Vocabulary Extension for \gls{HEA} data products.} \label{tab:dp_vocabulary} \end{longtable} \end{landscape} diff --git a/HighEnergyObsCoreExt.tex b/HighEnergyObsCoreExt.tex index b5fbd5c..4c36be3 100644 --- a/HighEnergyObsCoreExt.tex +++ b/HighEnergyObsCoreExt.tex @@ -43,6 +43,7 @@ \usepackage{amsmath} \usepackage{amssymb} \usepackage[nopostdot,style=super,nonumberlist,toc]{glossaries} +\usepackage{xurl} \usepackage{hyperref} % mireille : in order to flag changes to fill @@ -142,13 +143,13 @@ \section{Introduction} \pagebreak \section{High Energy Astrophysics Data} -\gls{HEA} data include observations obtained using photon detectors covering X-ray (from $\sim$0.1 keV to $\sim$120 keV) through gamma-ray (from 120 keV up to $\gtrsim$ PeV) energies, as well as cosmic-ray and astrophysical neutrino ($\gtrsim$ GeV) detectors, or other messengers related to \gls{HEA} phenomena. The domain is now sufficiently mature to provide open data that are science-ready and work with open analysis tools ({\em e.g.\/}, CIAO, \citealt{2026ApJ..1005..116F}, or Gammapy, \citealt{gammapy:2023}). The science output of the \gls{HEA} domain already includes advanced products such as images, cubes, spectra, and time series such as light curves and time-resolved spectra. Additional data products include fitted sky models with spatial, spectral, and/or temporal component(s), along with their confidence intervals or confidence limits, and covariance matrices. Finally, multiple \gls{HEA} instruments produce source catalogs and surveys covering up to the full the sky, which include maps of photon or particle flux, exposure, sensitivity, and aperture-photometry likelihood profiles. +\gls{HEA} data include observations obtained using photon detectors covering X-ray (from $\sim$0.1 keV to $\sim$120 keV) through gamma-ray (from 120 keV up to $\gtrsim$ PeV) energies, as well as cosmic-ray and astrophysical neutrino ($\gtrsim$ GeV) detectors, or other messengers related to \gls{HEA} phenomena. The domain is now sufficiently mature to provide open data that are science-ready and work with open analysis tools ({\em e.g.\/}, CIAO, \citealt{2026ApJ..1005..116F}, or Gammapy, \citealt{gammapy:2023}). The science output of the \gls{HEA} domain already includes advanced products such as images, cubes, spectra, and time series such as light curves and time-resolved spectra. Additional data products include fitted sky models with spatial, spectral, and/or temporal component(s), along with their confidence intervals or confidence limits, and covariance matrices. Finally, multiple \gls{HEA} instruments produce source catalogs and surveys covering up to the full sky, which include maps of photon or particle flux, exposure, sensitivity, and aperture-photometry likelihood profiles. Observations of the universe at the highest energies are based on techniques that are radically different compared to the UV through radio domains. \gls{HEA} observatories\footnote{For example, Chandra, XMM-Newton, Fermi, H.E.S.S., MAGIC, VERITAS, HAWC, LHAASO, IceCube, ANTARES, Auger, and soon CTAO, KM3NeT, and SWGO.} are generally designed to detect particles ({\em e.g.\/}, individual photons, cosmic-rays, or neutrinos) with the ability to estimate multiple observables for those particles. These detection techniques all rely on {\em event counting\/}\footnote{As opposed to signal integrating ({\em e.g.\/}, using a detector that accumulates the total photon signal during an exposure).}, where an event has some probability of being due to the interaction of a particle from an astrophysical source with the detectors, but also has some probability of being from instrumental or background effects. The data corresponding to an event are first an instrumental signal, which is then calibrated and processed to estimate physical quantities such as a time of arrival, point-of-origin on the sky, and an energy proxy associated with the event. Several other intermediate and qualifying characteristics may be associated with a detected event, depending on the detection technique. The ensemble of events detected over a given time interval and spatial field-of-view is referred to as an {\em event list\/}, which we designate an {\bf hea-event-list} in this document. -Though {\bf hea-event-list}s {\em may\/} include estimators for calibrated physical values, they typically still have to be corrected for the photometric, spectral, spatial, and/or temporal responses of the telescope and detector combination to yield scientifically interpretable information. The mappings between physical measurements of the source properties and the observables are called \glspl{IRF}\footnote{We try to avoid using the term \gls{IRF} in a normative sense since historical usage across the broad \gls{HEA} community (and from facility to facility) varies. In some cases, \gls{IRF} has been used to mean specifically the product of the \gls{ARF} and \gls{RMF}, whereas in other cases \gls{IRF} has been used more generally to mean any instrumental response function regardless of type.}. Some \glspl{IRF} are probabilistic in nature\footnote{For example, the energy redistribution matrix is a probability density function.}, and in addition may depend on the set of events selected for analysis by the end user. They are usually not invertible, so methods such as forward-folding fitting (using source models with any combination of spectral, spatial, temporal, and/or polarization components that are estimated) are needed to estimate physical properties, such as the true flux of particles from a source arriving at the instrument, given the measured observable quantities. The \glspl{IRF} generally evolve over time with the instrument and observation characteristics, and are usually defined for a specific time interval and may be decomposed into a standard set of independent components (see \S~3.1.5 of \citealt{2024ivoa.note.heig}), such as the spatial point-spread function or the energy-migration matrix or different messenger particle types, where each component may be stored or computed separately. Since both \glspl{IRF} and {\bf hea-event-list}s are required to analyze \gls{HEA} data, some \gls{IVOA} standards must be modified to expose both of them via the \gls{VO}. +Though {\bf hea-event-list}s {\em may\/} include estimators for calibrated physical values, they typically still have to be corrected for the photometric, spectral, spatial, and/or temporal responses of the telescope and detector combination to yield scientifically interpretable information. The mappings between physical measurements of the source properties and the observables are called \glspl{IRF}\footnote{We try to avoid using the term \gls{IRF} in a normative sense since historical usage across the broad \gls{HEA} community (and from facility to facility) varies. In some cases, \gls{IRF} has been used to mean specifically the product of the \gls{ARF} and \gls{RMF}, whereas in other cases \gls{IRF} has been used more generally to mean any instrumental response function regardless of type.}. Some \glspl{IRF} are probabilistic in nature\footnote{For example, the energy redistribution matrix is a probability density function.}, and in addition may depend on the set of events selected for analysis by the end user. They are usually not invertible, so methods such as forward-folding fitting (using source models with any combination of spectral, spatial, temporal, and/or polarization components that are estimated) are needed to estimate physical properties, such as the true flux of particles from a source arriving at the instrument, given the measured observable quantities. The \glspl{IRF} generally evolve over time with the instrument and observation characteristics, and are usually defined for a specific time interval and may be decomposed into a standard set of independent components (see \S~3.1.5 of \citealt{2024ivoa.note.heig}), such as the spatial point-spread function, the energy-migration matrix, or the telescope effective area, where each component may be stored or computed separately. Since both \glspl{IRF} and {\bf hea-event-list}s are required to analyze \gls{HEA} data, some \gls{IVOA} standards must be modified to expose both of them via the \gls{VO}. -In sections 3--5 of this document, we discuss enhancements to current \gls{IVOA} standards that are needed to support meaningful discovery of \gls{HEA} data, and make rationale and well-considered proposals for these enhancements based on our \gls {HEA} domain knowledge. The current ObsCore standard will be discussed in \S~\ref{sec:obscore}, focusing on attributes that need to be modified. Then, we propose the creation of a \gls{HEA} extension of ObsCore in \S~\ref{sec:obscoreext}, as some attributes are very specific to our domain. In these two sections, the discussion focuses on the attribute definitions rather than on the attribute values. In \S~\ref{sec:voc}, vocabulary enhancements are proposed for some ObsCore attributes, DataLink semantics, UCDs, and MIME-types. Finally, in \S~\ref{sec:accessoptions} we discuss various options for accessing \gls{HEA} data products using ObsTAP services. Appendix~\ref{sec:uc} provides a list of use case examples drawn from the X-ray, Gamma Ray, Cherenkov, and particle astrophysics communities. +In sections 3--5 of this document, we discuss enhancements to current \gls{IVOA} standards that are needed to support meaningful discovery of \gls{HEA} data, and make rational and well-considered proposals for these enhancements based on our \gls {HEA} domain knowledge. The current ObsCore standard will be discussed in \S~\ref{sec:obscore}, focusing on attributes that need to be modified. Then, we propose the creation of a \gls{HEA} extension of ObsCore in \S~\ref{sec:obscoreext}, as some attributes are specific to our domain. In these two sections, the discussion focuses on the attribute definitions rather than on the attribute values. In \S~\ref{sec:voc}, vocabulary enhancements are proposed for some ObsCore attributes, DataLink semantics, UCDs, and MIME-types. Finally, in \S~\ref{sec:accessoptions} we discuss various options for accessing \gls{HEA} data products using ObsTAP services. Appendix~\ref{sec:uc} provides a list of use case examples drawn from the X-ray, Gamma Ray, Cherenkov, and particle astrophysics communities. \pagebreak \section{ObsCore Attribute Definitions for High Energy Astrophysics Data} @@ -158,7 +159,7 @@ \section{ObsCore Attribute Definitions for High Energy Astrophysics Data} % Currently, some ObsCore attributes ({\em dataproduct\_type\/} and {\em calib\_level\/}) are formally defined in the ObsCore Recommendation Version 1.1 \citep{2017ivoa.spec.0509L} and also in the vocabularies documents \citep{2023ivoa.spec.0206D, 2021ivoa.spec.0525D}\footnote{Primarily the Data Product Type Vocabulary, \url{https://www.ivoa.net/rdf/product_type}.}, which may be referenced in future versions of the ObsCore Recommendation. For completeness, we are proposing in this document modifications to both the existing ObsCore Recommendation and IVOA vocabularies. -ObsCore attributes are defined explicitly in the ObsCore Recommendation Version 1.1 \citep{2017ivoa.spec.0509L}, which is the current \gls{IVOA} approved document that defines the ObsCore standard. This document proposes both \gls{HEA}-specific extensions to ObsCore, as well as some adjustments to the ObsCore standard. While we recognize that there are ongoing discussions within the \gls{IVOA} that will eventually lead to future versions of the ObsCore Recommendation, this document primarily uses the ObsCore Recommendation Version 1.1 as a basis as there is currently no referenceable document that captures proposed changes that will be included in next version of the ObsCore Recommendation. Once an updated ObsCore Recommendation becomes available, this document should be updated for consistency. We are aware that some of this work is focused on use of vocabularies documents \citep{2023ivoa.spec.0206D, 2021ivoa.spec.0525D}\footnote{Primarily the Data Product Type Vocabulary, \url{https://www.ivoa.net/rdf/product_type}.} for some ObsCore attributes ({\em e.g.\/}, {\em dataproduct\_type\/} and {\em calib\_level\/}) rather than defining those values explicitly in the ObsCore Recommendation. For completeness, we are proposing in this document modifications to both the existing ObsCore Recommendation and the IVOA vocabularies where appropriate. +ObsCore attributes are defined explicitly in the ObsCore Recommendation Version 1.1 \citep{2017ivoa.spec.0509L}, which is the current \gls{IVOA} approved document that defines the ObsCore standard. This document proposes both \gls{HEA}-specific extensions to ObsCore, as well as some adjustments to the ObsCore standard. While we recognize that there are ongoing discussions within the \gls{IVOA} that will eventually lead to future versions of the ObsCore Recommendation, this document primarily uses the ObsCore Recommendation Version 1.1 as a basis as there is currently no referenceable document that captures proposed changes that will be included in the next version of the ObsCore Recommendation. Once an updated ObsCore Recommendation becomes available, this document should be updated for consistency. We are aware that some of this work is focused on use of vocabularies documents \citep{2023ivoa.spec.0206D, 2021ivoa.spec.0525D}\footnote{Primarily the Data Product Type Vocabulary, \url{https://www.ivoa.net/rdf/product_type}.} for some ObsCore attributes ({\em e.g.\/}, {\em dataproduct\_type\/} and {\em calib\_level\/}) rather than defining those values explicitly in the ObsCore Recommendation. For completeness, we are proposing in this document modifications to both the existing ObsCore Recommendation and the IVOA vocabularies where appropriate. \subsection{{\em dataproduct\_type}} \label{sec:dataproduct_type} @@ -185,11 +186,11 @@ \subsection{{\em dataproduct\_type}} In addition to {\em dataproduct\_type\/} terms that focus on event data, we note that existing ObsCore definitions do not adequately span the breadth of ``advanced data products'' (typically with {\em calib\_level\/} $\ge$ 3) that may be generated from astronomical observations by users or observatories. The computational complexity of analyzing \gls{HEA} data robustly in the extreme Poisson regime ({\em e.g.\/}, Bayesian X-ray aperture photometry applied simultaneously to multiple overlapping detections and observations, or frequentist adjustment of models of electron populations for multi-wavelength data spanning from X-rays to PeV gamma rays) means that data providers may choose to provide such advanced data products\footnote{We choose to use the term ``advanced data products'' for these types of data products because that is the term for such data products, for example {\bf sed}, used in the ObsCore Recommendation document Version 1.1 ({\em e.g.\/}, \S~3.3.1). The Recommendation (\S~3.3.2) seems to imply that ``Enhanced Data Products'' have $\hbox{\em calib\_level\/} = 3$ and ``Analysis Data Products'' have $\hbox{\em calib\_level\/} = 4$, while ``Advanced Data Products'' may correspond to either {\em calib\_level\/} (\S~4.4).} directly to the end user. For example, the Chandra Source Catalog includes 38 types of advanced data products (for a total of $\sim\!90$ million files) and $\sim\!50$\% of these data product types are not well represented by a {\em dataproduct\_type\/} value that allows for meaningful data discovery. Users will certainly want to discover these data products independently from the associated progenitor observation data (and many of these data products combine data from multiple observations). We therefore propose the following additional {\em dataproduct\_type\/} (or {\em dataproduct\_subtype\/}) terms for these advanced data products, and note that these terms will certainly be useful independent of waveband ({\em i.e.\/}, they can be equally applicable to UV/optical, IR, and radio datasets): \begin{quote} -{\bf draws}: a dataset that records statistical draws computed from a probability distribution or a sample population, for example Markov chain Monte Carlo (MCMC) draws used when computing the Bayesian marginal probability density function for a random variable, or the DeltaTS associated with a quantity from a frequentist analysis. The draws can be interpreted to provide a robust estimation of the probability distribution of variable, and correlations between the draws provide information about how well the draws converge to the parent probability distribution.\footnote{As an example, within the standard $\Lambda CDM$ cosmological model, estimates of the cosmological density parameters $\Omega M$ and $\Omega\Lambda$ can be derived from the intersection of confidence contours from Hubble diagram of quasars with those from the Type Ia supernovae \citep{2019adds.book..283C}. These contours are {\bf draws}.} +{\bf draws}: a dataset that records statistical draws\footnote{A {\em statistical draw\/} is the formal process of sampling a random variable from a defined probability distribution, or selecting an element from a sample population that is distributed according to a defined probability distribution.} computed from a probability distribution or a sample population, for example Markov chain Monte Carlo (MCMC) draws used when computing the Bayesian marginal probability density function for a random variable, or the DeltaTS associated with a quantity from a frequentist analysis. The statistical draws can be interpreted to provide a robust estimation of the probability distribution of variable, and correlations between the draws provide information about how well the draws converge to the parent probability distribution.\footnote{As an example, within the standard $\Lambda CDM$ cosmological model, estimates of the cosmological density parameters $\Omega M$ and $\Omega\Lambda$ can be derived from the intersection of confidence contours from Hubble diagram of quasars with those from the Type Ia supernovae \citep{2019adds.book..283C}. These contours are {\bf draws}.} {\bf pdf}: a dataset that records the probability density function of a quantity, for example the Bayesian marginal probability density function for a random variable, or the DeltaTS associated with a quantity from a frequentist analysis. The probability density function provides a robust estimation of the variable and allows arbitrary confidence intervals to be computed directly from the distribution. -{\bf region}: a dataset that includes an encoding of (one or more) regions of parameter space, for example a spatial region or a region of phase space covered by a dataset. The set of dimensions represented by the region can be arbitrary.\footnote{One possible encoding is a \gls{MOC}; however the vast majority of pre-existing region data products in \gls{HEA} data archives currently use other encodings.} +{\bf region}: a dataset that includes an encoding of (one or more) regions of parameter space, for example a spatial region or a region of phase space covered by a dataset. The set of dimensions represented by the region can be arbitrary.\footnote{One possible encoding is a \gls{MOC}; however the vast majority of pre-existing region data products in \gls{HEA} data archives currently use other encodings.}$^{, }$\footnote{We note in passing that {\bf region} data products are not specific to \gls{HEA} but are also commonly used in other wavebands.} {\bf response-function}: a dataset that records a mapping from a physical quantity to an observable quantity. For \gls{HEA}, this may be the components of the composite \gls{IRF} such as an Auxiliary Response File ({\bf arf}), Redistribution Matrix File ({\bf rmf}), Effective Area ({\bf aeff}), Energy Dispersion ({\bf edisp}), or the Background Rate ({\bf bkgrate}). The Point Spread Function ({\bf psf}) is a response function that is generally applicable across multiple wavebands. While these datasets may generally be represented as an $N$-dimensional data cube, designating them as {\bf response-function}s enhances data discovery for very common types of \gls{HEA} datasets (see the use cases in Appendix~\ref{sec:uc}). \end{quote} @@ -217,7 +218,7 @@ \subsection{{\em calib\_level}} ObsCore defines calibration {\bf Level 1} as ``Instrumental data in a standard format (FITS, VOTable, SDFITS, ASDM, etc.) which could be manipulated with standard astronomical packages.'' and {\bf Level 2} as ``Calibrated, science ready data with the instrument signature removed.'' -However, some {\bf hea-event-list}s include spatial and time axes that are calibrated physical quantities, but the spectral axis is instrumental and requires application of the IRFs to remove this signature. This is typically done because the {\bf response-function}s can depend on the choice of region (spatial/time) from which the events are extracted (especially for telescope/detector combinations where the telescope position dithers on the sky during the exposure), which depends on the specific science case and therefore cannot be determined {\em a priori\/}. Such {\bf hea-event-list}s fall ``between'' {\em calib\_level\/} 1 and 2. +However, some {\bf hea-event-list}s include spatial and time axes that are calibrated physical quantities\footnote{In the sense that they have well-defined World Coordinate Systems.}, but the spectral axis is instrumental and requires application of the IRFs to remove this signature. This is typically done because the {\bf response-function}s can depend on the choice of region (spatial/time) from which the events are extracted (especially for telescope/detector combinations where the telescope position dithers on the sky during the exposure), which depends on the specific science case and therefore cannot be determined {\em a priori\/}. Such {\bf hea-event-list}s fall ``between'' {\em calib\_level\/} 1 and 2. On the other hand, other {\bf hea-event-list}s may not have any calibrated axes or may have all axes calibrated, and it is important to be able to differentiate between these for data discovery. While the value for {\em calib\_level\/} for any data product is left for the data provider to determine, we suggest that individual data providers set {\em calib\_level\/} = 1 if an {\bf hea-event-list} is considered to be ``uncalibrated'' according to normal usage for their data products, and set {\em calib\_level\/} = 2 if an {\bf hea-event-list} is considered to be ``calibrated'' according to normal usage for their data products. @@ -239,7 +240,7 @@ \subsection{{\em s\_ra\/}/{\em s\_dec}} We propose that the attributes {\em s\_ra\/}/{\em s\_dec\/} be redefined to be the ICRS right ascension and ICRS declination of ``a reference position (typically the center)'' of an observation on the sky, rather than the ICRS right ascension and ICRS declination of ``the center'' of the observation. For some facilities, the center (RA, Dec) may have a specific meaning (such as the location of the optical axis of the telescope), which often is not useful for advanced data products that may be extracted from a cut-out from the progenitor observation. Some facilities also allow an instrument to be displaced from the center of the focal plane, which means that the definition of ``the center'' of an observation may be unclear (especially when not tracking at sidereal rate or for facilities for which the PSF varies strongly across the telescope field of view). Since these cases effectively displace the observation field-of-view, ObsCore attributes such as {\em s\_fov\/} that are implicitly referenced to ({\em s\_ra\/}, {\em s\_dec\/}) will continue to behave as expected using the revised definition. -For non-pointing instruments (which may include all-sky instruments such as KM3NeT or HAWC), these fields are poorly defined (as is the case, generally for observations that are drift scans). For the time duration of the observation, one can compute an effective center position of the exposure skymap and the maximum radius of the covered area ({\em i.e.\/}, for an all-sky instrument this would be $2\pi\,\rm Sr$ solid angle in Alt/Az, which can be converted into a rotated area in RA/Dec). However, the utility of such a characterization depends on both the duration of the observation and the use case, and can be assumed to describe the center of the exposure skymap for typical aggregated datasets with observation duration spanning weeks up to years. +For non-pointing instruments (which may include all-sky instruments such as KM3NeT or HAWC), these fields are poorly defined (as is the case, generally for observations that are drift scans\footnote{A {\em drift scan\/} is an observational method where the telescope remains stationary and the rotation of the Earth sweeps the sky across the telescope field-of-view.}). For the time duration of the observation, one can compute an effective center position of the exposure skymap and the maximum radius of the covered area ({\em i.e.\/}, for an all-sky instrument this would be $2\pi\,\rm Sr$ solid angle in Alt/Az, which can be converted into a rotated area in RA/Dec). However, the utility of such a characterization depends on both the duration of the observation and the use case, and can be assumed to describe the center of the exposure skymap for typical aggregated datasets with observation duration spanning weeks up to years. \subsection{{\em s\_calib\_status}} @@ -329,7 +330,7 @@ \subsection{{\em tracking\_mode}} \item \texttt{solar-system-object-tracking}: observations pointed at a moving target, like the moon or other solar system bodies; \item \texttt{none}: observations with no telescope tracking. \end{itemize} -We intend that the name of this attribute harmonizes with the name of the similar attribute in the proposed ``IVOA Obscore Extension for Radio Data''\footnote{\url{https://github.com/ivoa-std/ObsCoreExtensionForRadioData}.} and note that the first three predefined values for {\em tracking\_mode\/} also harmonize with that proposal. The \texttt{none} {\em tracking\_mode\/} describes observations obtained while the telescope is not tracking ({\em e.g.\/}, observations obtained while the telescope is slewing). We further note that permissible {\em tracking\_mode\/} values may vary from facility to facility and from instrument to instrument and additional values beyond the predefined values are possible. This attribute is not expected to be used for non-pointing instruments. +We intend that the name of this attribute harmonizes with the name of the similar attribute in the proposed ``IVOA Obscore Extension for Radio Data''\footnote{\url{https://github.com/ivoa-std/ObsCoreExtensionForRadioData}.} and note that the first three predefined values for {\em tracking\_mode\/} also harmonize with that proposal. The \texttt{none} {\em tracking\_mode\/} describes observations obtained while the telescope is not tracking ({\em e.g.\/}, observations obtained while the telescope is slewing). While most of the {\em tracking\_mode\/}s are generally applicable to both ground- and space-based facilities, we note that the \texttt{fixed-az-el-transit} {\em tracking\_mode\/} is only useful for ground-based telescopes that use Alt/Az mounts. We further note that permissible {\em tracking\_mode\/} values may vary from facility to facility and from instrument to instrument and additional values beyond the predefined values are possible. This attribute is not expected to be used for non-pointing instruments. \subsection{{\em scan\_mode}} @@ -342,18 +343,18 @@ \subsection{{\em scan\_mode}} \item \texttt{on-the-fly-cross-map}: observations along parallel directions ({\em e.g.\/}, a wobble observation for \glspl{IACT}); \item \texttt{slew} : observations taken while the telescope is slewing. \end{itemize} -We intend that the name of this attribute harmonizes with the name of the similar attribute in the proposed ``IVOA Obscore Extension for Radio Data''\footnote{\url{https://github.com/ivoa-std/ObsCoreExtensionForRadioData}.} and note that the first five predefined values for {\em scan\_mode\/} also harmonize with that proposal. The \texttt{slew} {\em scan\_mode\/} describes observations obtained while the telescope is slewing ({\em i.e.\/}, where the field of view moves with arbitrary direction and speed while the telescope is repositioning), which is a mode used extensively by some satellite-based \gls{HEA} facilities. We further note that permissible {\em scan\_mode\/} values may vary from facility to facility and from instrument to instrument and additional values beyond the predefined values are possible. This attribute is not expected to be used for non-pointing instruments. +We intend that the name of this attribute harmonizes with the name of the similar attribute in the proposed ``IVOA Obscore Extension for Radio Data''\footnote{\url{https://github.com/ivoa-std/ObsCoreExtensionForRadioData}.} and note that the first five predefined values for {\em scan\_mode\/} also harmonize with that proposal. The \texttt{slew} {\em scan\_mode\/} describes observations obtained while the telescope is slewing ({\em i.e.\/}, where the field of view moves with arbitrary direction and speed while the telescope is repositioning), which is a mode used extensively by some satellite-based \gls{HEA} facilities. We further note that permissible {\em scan\_mode\/} values may vary from facility to facility and from instrument to instrument and additional values beyond the predefined values are possible. This attribute is not expected to be used for non-pointing instruments. -\subsection{{\em pointing\_mode}} +\subsection{{\em array\_pointing\_mode}} -Some \gls{HEA} facilities can obtain observations using multiple telescope simultaneously with the individual telescopes pointing in slightly different directions. For example, for the \glspl{IACT} telescope array, the individual telescope pointings may be in the same direction, convergent, or divergent. We propose to add an optional attribute {\em pointing\_mode\/}, with a default value of ``NULL'' to handle \gls{WCD} and neutrino instruments, to distinguish these modes. Constraints on {\em pointing\_mode\/} can provide a simple way to discover data sets for a specific facility/instrument combination. We propose predefined {\em pointing\_mode\/} values for \gls{HEA} data include the following: +Some \gls{HEA} facilities can obtain observations using multiple telescope simultaneously with the individual telescopes pointing in slightly different directions. For example, for the \glspl{IACT} telescope array, the individual telescope pointings may be in the same direction, convergent, or divergent. We propose to add an optional attribute {\em array\_pointing\_mode\/}, with a default value of ``NULL'' to handle \gls{WCD} and neutrino instruments, to distinguish these modes. Constraints on {\em array\_pointing\_mode\/} can provide a simple way to discover data sets for a specific facility/instrument combination. We propose predefined {\em array\_pointing\_mode\/} values for \gls{HEA} data include the following: \begin{itemize} \item \texttt{parallel}: the telescopes are all pointing in the same direction; \item \texttt{convergent[-ang]}: the telescope pointings are convergent; \item \texttt{divergent[-ang]}: the telescope pointings are divergent. \end{itemize} -The optional suffix ``[-ang]'' specifies a reference angle for a convergent or divergent pointing, for example ``convergent-5d''. We note that permissible {\em pointing\_mode\/} values may vary from facility to facility and from instrument to instrument. This attribute is not expected to be used for non-pointing instruments. +The optional suffix ``[-ang]'' specifies a reference angle for a convergent or divergent pointing, for example ``convergent-5d''. We note that permissible {\em array\_pointing\_mode\/} values may vary from facility to facility and from instrument to instrument. This attribute is not expected to be used for non-array or non-pointing instruments. \subsection{{\em analysis\_mode}} @@ -372,7 +373,7 @@ \subsection{{\em event\_type}} \subsection{{\em messenger\_name}} \label{sec:messenger} -While many \gls{HEA} facilities detect photons ({\em e.g.\/}, most space-based X-ray and gamma-ray facilities), other facilities can detect alternate messenger particles, such as cosmic-rays, neutrinos, or massive particles. The end user will therefore need the ability to select the messenger when querying for data products using ObsCore. +While many \gls{HEA} facilities detect photons ({\em e.g.\/}, most space-based X-ray and gamma-ray facilities), other facilities can detect alternate messenger particles, such as cosmic-rays, neutrinos, or massive particles. Even a single \gls{HEA} facility can produce data for multiple different messengers. For example, \gls{HESS} and \gls{IACT} are nominally gamma-ray instruments, but also have detected other messengers such as electrons, protons, and iron nuclei. The end user will therefore need the ability to select the messenger when querying for data products using Obs\-Core. We propose to add an optional attribute {\em messenger\_name\/} that specifies the {\em name} of the messenger type for an observation. Constraints on {\em messenger\_name\/} can provide a simple way to discover data sets for a specific messenger. While the entries in the IVOA Messengers Vocabulary\footnote{\url{https://www.ivoa.net/rdf/messenger}} provide a useful starting point for populating this attribute, we propose that the predefined {\em messenger\_name\/} values for \gls{HEA} data include at least the following: \begin{itemize} @@ -406,8 +407,8 @@ \subsection{Summary} Table~\ref{tab:obscore_hea} summarizes the proposal for the \gls{HEA} extension of ObsCore. We use the table name {\em ivoa.obscore\_hea\/} to describe the extension table here and in Appendix~\ref{sec:uc}. In addition to the table columns identified herein, we note that the {\em ivoa.obscore\_hea\/} table must also include the {\em obs\_publisher\_did\/} column that is shared exactly with the {\em ivoa.obscore\/} table so that a {\tt NATURAL JOIN} will yield one-to-one matched per-dataset rows of ObsCore and \gls{HEA} extension metadata. \begin{landscape} -\begin{center} -\begin{longtable}{ | m{0.15\linewidth} | m{0.23\linewidth} | m{0.07\linewidth} | m{0.07\linewidth} | m{0.4\linewidth} | m{0.05\linewidth} |} +%\begin{center} +\begin{longtable}{ | m{0.175\linewidth} | m{0.23\linewidth} | m{0.07\linewidth} | m{0.07\linewidth} | m{0.375\linewidth} | m{0.05\linewidth} |} \hline {\centering \bf Column Name} &{\centering \bf UCD} &{\centering \bf Unit} &{\centering \bf Type} &{\centering \bf Description} &{\centering \bf MAN}\\ @@ -436,7 +437,7 @@ \subsection{Summary} \hline {\em scan\_mode\/} & \ucd{meta.code;obs.param} & unitless & string &{Scan mode of an observation for pointing instruments ({\em e.g.\/}, on-sources, on-off, raster-map, on-the-fly-cross-scan, on-the-fly-cross-map, slew)} & NO \\ \hline -{\em pointing\_mode\/} & \ucd{meta.code;obs.param} & unitless & string &{Pointing mode of an observation for pointing instruments ({\em e.g.\/}, parallel, convergent[-ang], divergent[-ang])} & NO \\ +{\em array\_pointing\_mode\/} & \ucd{meta.code;obs.param} & unitless & string &{Pointing mode of an observation for array instruments ({\em e.g.\/}, parallel, convergent[-ang], divergent[-ang])} & NO \\ \hline {\em analysis\_mode\/} & \ucd{meta.code;obs.param} & unitless & string &{Data reduction/analysis mode}& NO \\ \hline @@ -450,7 +451,6 @@ \subsection{Summary} \label{tab:obscore_hea} \end{longtable} %\end{center} -\end{center} \end{landscape} \pagebreak @@ -499,7 +499,7 @@ \subsubsection{Response Functions} The following data product types specialize {\bf response function}. Note that while most of these are primarily used in \gls{HEA}, the point spread function ({\bf psf}) is a {\bf response-function} that is generally applicable across multiple wavebands. We again note that {\bf response-function}s for some facilities and instruments may not be separable into these distinct components. \begin{quote} -{\bf aeff}: A dataset that records the ``effective area'' of a telescope and/or instrument. The effective area is the geometric area of the telescope and/or instrument reduced by efficiency factors such as reflectivity and vignetting, among other effects \citep{deil_2022_7304668}. +{\bf aeff}: A dataset that records the ``effective area'' of a telescope and/or instrument. The effective area is the geometric area\footnote{The ``geometric'' area may be difficult to determine for some telescopes; instead the true collection area may need to be computed using techniques such as Monte-Carlo integration to account for spatial and/or energy-dependent instrumental effects.} of the telescope and/or instrument reduced by efficiency factors such as reflectivity and vignetting, among other effects \citep{deil_2022_7304668}. {\bf arf}: A dataset that records the combined telescope/instrument effective area and detector quantum efficiency as a function of energy \citep{ogip_spectrum_1998}. @@ -521,7 +521,7 @@ \subsubsection{Advanced Data Products} Users will certainly want to discover these data products independently from the associated progenitor observation data (and many of these data products combine data from multiple observations). We therefore propose the following additional data product types for these advanced data products, and note that these data product types will certainly be useful independent of waveband ({\em i.e.\/}, they can be equally applicable to UV/optical, IR, and radio datasets): \begin{quote} - {\bf draws}: A dataset that records statistical draws computed from a probability distribution or a sample population, for example Markov chain Monte Carlo (MCMC) draws used when computing the Bayesian marginal probability density function for a random variable, or the DeltaTS associated with a quantity from a frequentist analysis. The draws can be interpreted to provide a robust estimation of the probability distribution of variable, and correlations between the draws provide information about how well the draws converge to the parent probability distribution. + {\bf draws}: A dataset that records statistical draws computed from a probability distribution or a sample population, for example Markov chain Monte Carlo (MCMC) draws used when computing the Bayesian marginal probability density function for a random variable, or the DeltaTS associated with a quantity from a frequentist analysis. The statistical draws can be interpreted to provide a robust estimation of the probability distribution of variable, and correlations between the draws provide information about how well the draws converge to the parent probability distribution. {\bf pdf}: A dataset that records the probability density function of a quantity, for example the Bayesian marginal probability density function for a random variable. The probability density function provides a robust estimation of the variable and allows arbitrary confidence intervals to be computed directly from the distribution. diff --git a/Makefile b/Makefile index afcea7a..e6e66b0 100644 --- a/Makefile +++ b/Makefile @@ -8,7 +8,7 @@ DOCNAME = HighEnergyObsCoreExt DOCVERSION = 1.0 # Publication date, ISO format; update manually for "releases" -DOCDATE = 2026-08-31 +DOCDATE = 2026-09-01 # What is it you're writing: NOTE, WD, PR, REC, PEN, or EN DOCTYPE = PEN diff --git a/UseCases.tex b/UseCases.tex index 0fcd182..7adc5f1 100644 --- a/UseCases.tex +++ b/UseCases.tex @@ -140,7 +140,7 @@ \subsubsection{Use Case --- Search for event lists and their \glspl{IRF} of CTAO \item energy\_max >= $10^{13}$. \end{enumerate} -First, run the ObCore query: +First, run the ObsCore query: \begin{verbatim} SELECT * FROM ivoa.obscore NATURAL JOIN ivoa.obscore_hea diff --git a/extendedAccessTonewtypesOfproducts.tex b/extendedAccessTonewtypesOfproducts.tex index c9af5b5..5b5e7b5 100644 --- a/extendedAccessTonewtypesOfproducts.tex +++ b/extendedAccessTonewtypesOfproducts.tex @@ -217,9 +217,9 @@ \subsection{Accessing Datasets via an Alternate Table Joined with the {\tt ivoa. More generally, the cardinality of the relationship between a {\bf response-function} dataset and an {\bf hea-event-list} dataset will vary according to the facility and may be one-to-one, one-to-many, many-to-one, or even many-to-many. Compared to the set of queryable attributes included in {\tt ivoa.obscore} table, typically more but occasionally fewer attributes will be required to select an appropriate {\bf response-function} dataset of a given type. The set of attributes required to identify an appropriate {\bf response-function} dataset will typically depend on the type of {\bf response-function}. -Using this approach, {\bf response-function} data products could be described by a {\tt ivoa.response} table . +Using this approach, {\bf response-function} data products could be described by an {\tt ivoa.response} table. %{\tt ivoa.response$\{$\_xxx$\}$} tables, where {\tt $\{$\_xxx$\}$} is optional and would depend on the type of {\bf response-function} in the case that different sets of queryable attributes are required for different types of {\bf response-function}s. -A conceptual example of such a {\tt ivoa.response} table is presented in Appendix~\ref{sec:accessoptionsappendix}. +A conceptual example of such an {\tt ivoa.response} table is presented in Appendix~\ref{sec:accessoptionsappendix}. A similar strategy could apply for discovering data products for an observation that are associated with an {\bf hea-event-list} data product if those data products require a set of queryable attributes that is not present in the {\tt ivoa.obscore} table. A dedicated table would allow more specific properties of these data products to be listed in optional table columns ({\em e.g.\/}, if certain data products have dependencies on telescope altitude and azimuth, or off-axis and azimuthal angles). Of course, this approach would preclude them from being directly queryable in the {\tt ivoa.obscore} table, which may be undesirable from the user perspective. diff --git a/extendedAccessTonewtypesOfproductsAppendix.tex b/extendedAccessTonewtypesOfproductsAppendix.tex index 9ec0c52..2caf550 100644 --- a/extendedAccessTonewtypesOfproductsAppendix.tex +++ b/extendedAccessTonewtypesOfproductsAppendix.tex @@ -2,9 +2,9 @@ In order to handle the various possible cardinality relationships between {\bf response-function} and {\bf hea-event-list} datasets, foreign keys must be defined in the {\tt ivoa.response} tables that will allow {\tt JOIN} operations between those tables and the {\tt ivoa.obscore} table. -The {\tt ivoa.response} table uses a specific resp\_dataproduct\_type column which is compliant to the response type vocabulary : \scriptsize{\url{https://www.ivoa.net/rdf/response-type/}} and has its own resp\_obs\_publisher\_did column as well, in order to distinguish which response matches which ObsCore data set in the {\tt ivoa.response} table. +The {\tt ivoa.response} table uses a specific {\em resp\_dataproduct\_type\/} column that is compliant with the proposed response type vocabulary (\url{https://www.ivoa.net/rdf/response-type/}) and has its own {\em resp\_obs\_publisher\_did\/} column as well, in order to distinguish which response matches which ObsCore data set in the {\tt ivoa.response} table. -\subsection*{Proposal for a Response Table {\tt ivoa.response}} +\subsection*{Preliminary Proposal for a Response Table {\tt ivoa.response}} {\small \begin{longtable}{|m{0.25\linewidth}|m{0.1\linewidth}|m{0.10\linewidth}|m{0.41\linewidth}|} @@ -32,16 +32,16 @@ \subsection*{Proposal for a Response Table {\tt ivoa.response}} {\em resp\_energy\_min} & eV & double & Energy band minimal value for response use \\\hline {\em resp\_energy\_max} & eV & double & Energy band maximal value for response use \\\hline -\caption{Proposal for a standard response table : with appropriate further study the columns identified herein could form the basis for a recommended base set of columns for the {\tt ivoa.response} table.} +\caption{Preliminary proposal for a standard response table. With appropriate further study the columns identified herein could form the basis for a recommended base set of columns for an {\tt ivoa.response} table.} \label{tab:response_table} \end{longtable} } %\end{center} %\end{table} -\subsection*{Example queries} -\noindent Implementing an {\tt ivoa.response} table similar to this would allow queries such as the following: \\ -\subsubsection*{Retrieve psf response for {\em obs\_id} and position criterium } +\subsection*{Example Queries} +\noindent Implementing an {\tt ivoa.response} table similar to this would allow queries such as the following: +\subsubsection*{Retrieve {\bf psf} {\bf response-function}s for {\em obs\_id\/} and position criteria} \noindent Find all datasets satisfying: \begin{enumerate}[(i)] \item Position inside 3 arcmin from (83.6324, $+22.0174$), @@ -50,14 +50,13 @@ \subsubsection*{Retrieve psf response for {\em obs\_id} and position criterium } \item obs\_collection = ``HESS''. \end{enumerate} -When using the response table defined in Table \ref{tab:response_table}, we can join the {\tt ivoa.obscore} table with the {\tt ivoa.response} table on -one of the two keys {\em obs\_id\/} or {\em obs\_publisher\_did\/} for instance, as in: +When using the response table defined in Table \ref{tab:response_table}, we can join the {\tt ivoa.obscore} table with the {\tt ivoa.response} table on one of the two keys ({\em obs\_id\/} and {\em obs\_publisher\_did\/}), for example: {\small \begin{verbatim} SELECT o.obs_publisher_did, o.obs_id, s_ra, s_dec, s_fov, t_min, t_max, energy_min, energy_max, access_url, access_format, -resp_publisher_did, r.obs_publisher_did, r.obs_id, resp_access_url, resp_access_format, -resp_energy_max, resp_energy_min +resp_publisher_did, r.obs_publisher_did, r.obs_id, resp_access_url, +resp_access_format, resp_energy_max, resp_energy_min FROM ivoa.obscore as o NATURAL JOIN ivoa.response as r WHERE @@ -68,14 +67,14 @@ \subsubsection*{Retrieve psf response for {\em obs\_id} and position criterium } AND (o.obs_collection = 'HESS') \end{verbatim} } -If additional constraints are required, for instance on time, then we can add another set of columns in {\tt ivoa.response}({\em e.g.\/}) {\em resp\_t\_min\/}, and {\em resp\_t\_max\/} and constrain the time interval by adding a clause like: +If additional constraints are required, for instance on time, then we can add another set of columns in {\tt ivoa.response} ({\em e.g.\/}) {\em resp\_t\_min\/} and {\em resp\_t\_max\/}, and constrain the time interval by adding a clause like: {\small \begin{verbatim} AND (resp_t_min > 56000.0 and resp_t_max < 56001.5) \end{verbatim} } -\subsubsection*{Retrieve edisp response for data sets constrained by {\em obs\_id} or position and by {\em scan\_mode}} +\subsubsection*{Retrieve {\bf edisp} {\bf response-function}s for data sets constrained by {\em obs\_id} or position and by {\em scan\_mode}} {\small \begin{verbatim} SELECT o.obs_publisher_did, o.obs_id, s_ra, s_dec, s_fov, t_min, t_max, @@ -87,13 +86,15 @@ \subsubsection*{Retrieve edisp response for data sets constrained by {\em obs\_i NATURAL JOIN ivoa.obscore-hea WHERE (o.obs_collection = 'IACT') - AND (scan_mode=raster-map) + AND (scan_mode = 'raster-map') AND (o.obs_id = '1976') - OR CONTAINS(POINT(s_ra, s_dec), CIRCLE, 83.6324, +22.0174, 0.083333) = 1) as s + OR CONTAINS(POINT(s_ra, s_dec), CIRCLE, 83.6324, +22.0174, 0.083333) = 1) + as s JOIN ivoa.response ON -resp.obs_id =s.obs_id +resp.obs_id = s.obs_id WHERE (resp_product_type = 'edisp') -% s is the list of observations satisfying position or obs_id and scan mode criteria +% s is the list of observations satisfying position or obs_id and scan_mode +% criteria \end{verbatim} }% end \small