Source code for tom_dataservices.views

import logging
from requests import HTTPError
from requests.exceptions import ReadTimeout

from django_filters.views import FilterView
from django_filters import FilterSet, ChoiceFilter, CharFilter
from django.views.generic.edit import DeleteView, FormView
from django.views.generic.base import TemplateView, View
from django.contrib.auth.mixins import LoginRequiredMixin
from django.shortcuts import redirect, get_object_or_404
from django.urls import reverse, reverse_lazy

from django.utils import timezone

from django.core.cache import cache
from django.contrib import messages
from urllib.parse import urlencode

from tom_dataservices.models import DataServiceQuery
from tom_dataservices.dataservices import get_data_service_classes, get_data_service_class, NotConfiguredError
from tom_dataservices.dataservices import MissingDataException, QueryServiceError
from tom_dataservices.forms import UpdateDataFromDataServiceForm


logger = logging.getLogger(__name__)




class DataServiceQueryFilterSet(FilterSet):
    """
    Defines the available fields for filtering the list of queries.
    """
    data_service = ChoiceFilter(
        choices=[(k, k) for k in get_data_service_classes().keys()]
    )
    name = CharFilter(lookup_expr='icontains')

    class Meta:
        model = DataServiceQuery
        fields = ['data_service', 'name']





class DataServiceQueryListView(FilterView):
    """
    View that displays all saved ``DataServiceQuery`` objects.
    """
    model = DataServiceQuery
    template_name = 'tom_dataservices/query_list.html'
    filterset_class = DataServiceQueryFilterSet



    def get_context_data(self, *args, **kwargs):
        """
        Adds the data services available to the TOM to the context dictionary.

        :returns: context
        :rtype: dict
        """
        context = super().get_context_data(*args, **kwargs)
        context['installed_services'] = get_data_service_classes()
        return context






class DataServiceQueryCreateView(LoginRequiredMixin, FormView):
    """
    View for creating a new query to a data service. Requires authentication.
    """
    template_name = 'tom_dataservices/query_form.html'
    success_url = reverse_lazy('tom_dataservices:run')



    def get_data_service_name(self):
        """
        Returns the data service specified in the request

        :returns: DataService name
        :rtype: str
        """
        if self.request.method == 'GET':
            return self.request.GET.get('data_service')
        elif self.request.method == 'POST':
            return self.request.POST.get('data_service')




    def get_form_class(self):
        """
        Returns the form class to use in this view. The form class will be the one defined in the specific dataservice
        module for which a new query is being created.
        """
        data_service_name = self.get_data_service_name()

        if not data_service_name:
            raise ValueError('Must provide a data service name')

        return get_data_service_class(data_service_name).get_form_class()




    def get_form(self, form_class=None):
        """
        Returns an instance of the form to be used in this view.

        :returns: Form instance
        :rtype: django.forms.Form
        """
        form = super().get_form()
        form.helper.form_action = reverse('tom_dataservices:create')
        return form




    def get_initial(self):
        """
        Returns the initial data to use for forms on this view.

        :returns: dict of initial values
        :rtype: dict
        """
        initial = super().get_initial()
        initial['data_service'] = self.get_data_service_name()
        return initial




    def form_valid(self, form):
        """
        Saves the associated ``DataServiceQuery`` and redirects to the ``DataServiceQuery`` list.
        """

        if form.cleaned_data['query_save']:
            form.save()

        # Store final form data in the session so it can be retrieved in the run_query view.
        # This is how we pass the form data for unsaved queries without passing the entire form data as kwargs
        self.request.session['query_parameters'] = form.cleaned_data

        return redirect(self.success_url)




    def get_context_data(self, *args, **kwargs):
        """
        Adds any form partials to the context.

        :returns: context
        :rtype: dict
        """
        context = super().get_context_data()

        form = context['form']
        data_service_name = self.get_data_service_name()
        simple_form = form.get_simple_form_partial()
        advanced_form = form.get_advanced_form_partial()

        context['simple_fields'] = []
        if not simple_form and form.simple_fields():
            for field in form.simple_fields():
                context['simple_fields'].append(form[field])
            simple_form = 'tom_dataservices/partials/basic_simple_form.html'
        context['simple_form'] = simple_form
        context['advanced_form'] = advanced_form
        context['app_link'] = get_data_service_class(data_service_name).app_link
        context['app_version'] = get_data_service_class(data_service_name).app_version
        context['verbose_name'] = get_data_service_class(data_service_name).verbose_name
        context['info_url'] = get_data_service_class(data_service_name).info_url

        return context






class RunQueryView(TemplateView):
    """
    View that handles the running of a query that was either submitted via the form or saved as a ``DataServiceQuery``.
    """
    template_name = 'tom_dataservices/query_result.html'



    def get_context_data(self, *args, **kwargs):
        """
        Collects the query parameters from either a saved ``DataServiceQuery`` or from the session data,
        runs the query, and returns the results as context for the list template.

        :returns: context
        :rtype: dict
        """
        context = super().get_context_data()
        query = None
        query_feedback = ""
        data_service_class = None
        cached_results = {}
        query_parameters = {}

        # Do query and get query results
        try:
            # get the DataService class. Pull saved query if PK available, otherwise use session data.
            if self.kwargs.get('pk', None) is not None:
                query = get_object_or_404(DataServiceQuery, pk=self.kwargs['pk'])
                data_service_class = get_data_service_class(query.data_service)()
                query_parameters = data_service_class.build_query_parameters(query.parameters)
                query.last_run = timezone.now()
                query.save()
            else:
                input_parameters = self.request.session.get('query_parameters', {})
                data_service_class = get_data_service_class(input_parameters['data_service'])()
                query_parameters = data_service_class.build_query_parameters(input_parameters)
            # Check cached query is the same and pull cache if needed.
            if query_parameters == cache.get('query_params'):
                cached_results = cache.get_many([f'result_{result_id}' for result_id in range(0, 99)])
            else:
                cache.clear()
            if cached_results:
                results = [cached_results[key] for key in cached_results]
            else:
                results = data_service_class.query_targets(query_parameters)
        except HTTPError as e:
            results = iter(())
            query_feedback += f"Issue fetching query results, please try again.</br>{e}</br>"
        except NotConfiguredError as e:
            results = iter(())
            query_feedback += f"Configuration Error. Please contact your TOM Administrator: </br>{e}</br>"
        except QueryServiceError as e:
            results = iter(())
            query_feedback += f"There was an error with the underlying query service: </br>{e}</br>"
        except ReadTimeout as e:
            results = iter(())
            query_feedback += f"The query service connection timed out: </br>{e}</br>"

        # create context for template
        context['query'] = query
        context['query_feedback'] = query_feedback
        context['too_many_results'] = False
        context['data_service'] = data_service_class.name
        if data_service_class.query_results_table:
            context['query_results_table'] = data_service_class.query_results_table
        else:
            context['query_results_table'] = 'tom_dataservices/partials/query_results_table.html'

        context['results'] = []

        # Set Cache and context
        cache.set('query_params', query_parameters, 3600)
        try:
            for (i, result) in enumerate(results):
                if i > 99:
                    # issue 1172 too many alerts causes the cache to overflow
                    context['too_many_results'] = True
                    break
                result['id'] = i
                cache.set(f'result_{i}', result, 3600)
                context['results'].append(result)
        except StopIteration:
            pass

        # allow the Data Service to add to the context (besides the query_results)
        data_service_context_additions = data_service_class.get_additional_context_data()
        context |= data_service_context_additions

        return context






class DataServiceQueryDeleteView(LoginRequiredMixin, DeleteView):
    """
    View that handles the deletion of a saved ``DataServiceQuery``. Requires authentication.
    """
    model = DataServiceQuery
    success_url = reverse_lazy('dataservices:query_list')





class DataServiceQueryUpdateView(LoginRequiredMixin, FormView):
    """
    View that handles the modification of a previously saved ``DataServiceQuery``. Requires authentication.
    """
    template_name = 'tom_dataservices/query_form.html'
    success_url = reverse_lazy('tom_dataservices:run')
    object = None



    def get_object(self):
        """
        Returns the ``DataServiceQuery`` object that corresponds with the ID in the query path.

        :returns: ``DataServiceQuery`` object
        :rtype: ``DataServiceQuery``
        """
        return DataServiceQuery.objects.get(pk=self.kwargs['pk'])




    def get_form_class(self):
        """
        Returns the form class to use in this view. The form class will be the one defined in the specific data service
        module for which the query is being updated.
        """
        self.object = self.get_object()
        return get_data_service_class(self.object.data_service).get_form_class()




    def get_form(self, form_class=None):
        """
        Returns an instance of the form to be used in this view.

        :returns: Form instance
        :rtype: django.forms.Form
        """
        form = super().get_form()
        form.helper.form_action = reverse(
            'dataservices:update', kwargs={'pk': self.object.id}
        )
        return form




    def get_initial(self):
        """
        Returns the initial data to use for forms on this view. Initial data for this form consists of the name of
        the Data Service that the query is for and the saved query parameters.

        :returns: dict of initial values
        :rtype: dict
        """
        initial = super().get_initial()
        initial.update(self.object.parameters)
        initial['data_service'] = self.object.data_service
        return initial




    def form_valid(self, form):
        """
        Saves the associated ``DataServiceQuery`` if requested and redirects to the ``DataServiceQuery`` list.
        """
        if form.cleaned_data['query_save']:
            form.save(query_id=self.object.id)
        # Update session with form data so that we can run unsaved queries.
        self.request.session['query_parameters'] = form.cleaned_data
        return redirect(self.success_url)




    def get_context_data(self, *args, **kwargs):
        """
        Adds any form partials to the context.

        :returns: context
        :rtype: dict
        """
        context = super().get_context_data()

        simple_form = context['form'].get_simple_form_partial()
        advanced_form = context['form'].get_advanced_form_partial()
        context['simple_form'] = simple_form
        context['advanced_form'] = advanced_form
        context['object'] = self.object
        return context






class CreateTargetFromQueryView(LoginRequiredMixin, View):
    """
    View that handles the creation of ``Target`` objects from a Data Service Query result. Requires authentication.
    """



    def post(self, request, *args, **kwargs):
        """
        Handles the POST requests to this view. Creates a ``Target`` for each query result sent in the POST. Redirects
        to the ``TargetListView`` if multiple targets were created, and the ``TargetUpdateView`` if only one was
        created. Redirects to the ``RunQueryView`` if no ``Target`` objects were successfully created.
        """
        query_id = self.request.POST['query_id']
        data_service_name = self.request.POST['data_service']
        data_service_class = get_data_service_class(data_service_name)()
        results = self.request.POST.getlist('selected_results')
        errors = []
        target = None
        if not results:
            messages.warning(request, 'Please select at least one result from which to create a target.')
            if query_id:
                return redirect(reverse('dataservices:run_saved', kwargs={'pk': query_id}))
            else:
                return redirect(reverse('dataservices:run'))
        try:
            for result_id in results:
                cached_result = cache.get(f'result_{result_id}')
                if not cached_result:
                    messages.error(request, 'Could not create targets. Try re-running the query again.')
                    if query_id:
                        return redirect(reverse('dataservices:run_saved', kwargs={'pk': query_id}))
                    else:
                        return redirect(reverse('dataservices:run'))
                target = data_service_class.to_target(cached_result, request=request)
                # Do not attempt to store Reduced Datums if no Target.
                if target:
                    try:
                        data_service_class.to_reduced_datums(target, cached_result.get('reduced_datums'))
                    except MissingDataException:
                        try:
                            data = data_service_class.query_reduced_data(target)
                            data_service_class.to_reduced_datums(target, data)
                        except QueryServiceError as e:
                            messages.error(request, f'Error retrieving data from Data Service: {e}')
        except NotImplementedError as e:
            messages.error(request, str(e))
        if len(results) == len(errors):
            if query_id:
                return redirect(reverse('dataservices:run_saved', kwargs={'pk': query_id}))
            else:
                return redirect(reverse('dataservices:run'))
        if len(results) == 1 and target:
            return redirect(reverse('tom_targets:detail', kwargs={'pk': target.id}))
        return redirect(reverse('tom_targets:list'))




def update_data_from_query(request):

    if request.method == "POST":
        form = UpdateDataFromDataServiceForm(request.POST)
        data = {}
        if form.is_valid():
            target = form.cleaned_data['target']
            try:
                data_service_class = get_data_service_class(form.cleaned_data['data_service'])()
                data = data_service_class.query_reduced_data(target)
                data_service_class.to_reduced_datums(target, data)
                alias_data = data_service_class.query_aliases(target=target)
                data_service_class.to_aliases(target, alias_data)
            except QueryServiceError as e:
                messages.error(request, f'Error retrieving data from Data Service: {e}')

            # redirect to data page
            base_url = reverse('tom_targets:detail', kwargs={'pk': target.id})
            if 'photometry' in data.keys():
                page_filters = urlencode({'tab': 'photometry'})
            elif 'spectroscopy' in data.keys():
                page_filters = urlencode({'tab': 'spectroscopy'})
            else:
                page_filters = urlencode({'tab': 'manage-data'})
            return redirect(f'{base_url}?{page_filters}')
    return redirect('/')