Add new changes and Apis for extracts

Change latest prompt with used excerpts
Add OpenAiChat class for AzureOpenAI
Handle Empty df for Ops and recursive function for summary

api/drf_views.py

@@ -854,7 +854,12 @@ def list(self, request, *args, **kwargs):
 
 
 class AppealDocumentViewset(viewsets.ReadOnlyModelViewSet):
-    queryset = AppealDocument.objects.all()
+    queryset = AppealDocument.objects.select_related(
+        "type",
+        "iso",
+    ).prefetch_related(
+        "appeal__event__countries_for_preview",
+    )
     ordering_fields = (
         "created_at",
         "name",

api/filter_set.py

@@ -23,6 +23,11 @@
     Snippet,
 )
 from api.view_filters import ListFilter
+from per.models import (
+    OpsLearningCacheResponse,
+    OpsLearningComponentCacheResponse,
+    OpsLearningSectorCacheResponse,
+)
 
 
 class UserFilterSet(filters.FilterSet):
@@ -286,6 +291,16 @@ class AppealDocumentFilter(filters.FilterSet):
         widget=filters.widgets.CSVWidget,
         queryset=Appeal.objects.all(),
     )
+    # NOTE: Filters are used to get documents of used ops learning
+    insight_id = filters.NumberFilter(
+        label="Base Insight id for source document",
+        method="get_cache_base_document",
+    )
+    insight_sector_id = filters.NumberFilter(label="Sector insight id for source document", method="get_cache_sector_document")
+    insight_component_id = filters.NumberFilter(
+        label="Component insight id for source document",
+        method="get_cache_component_document",
+    )
 
     class Meta:
         model = AppealDocument
@@ -299,6 +314,27 @@ def get_appeal_filter(self, qs, name, value):
             return qs.filter(appeal__in=value).distinct()
         return qs
 
+    def get_cache_base_document(self, qs, name, value):
+        if value and (ops_learning_cache_response := OpsLearningCacheResponse.objects.filter(id=value).first()):
+            return qs.filter(id__in=ops_learning_cache_response.used_ops_learning.values_list("appeal_document_id", flat=True))
+        return qs
+
+    def get_cache_sector_document(self, qs, name, value):
+        if value and (ops_learning_sector_cache_response := OpsLearningSectorCacheResponse.objects.filter(id=value).first()):
+            return qs.filter(
+                id__in=ops_learning_sector_cache_response.used_ops_learning.values_list("appeal_document_id", flat=True)
+            )
+        return qs
+
+    def get_cache_component_document(self, qs, name, value):
+        if value and (
+            ops_learning_component_cache_response := OpsLearningComponentCacheResponse.objects.filter(id=value).first()
+        ):
+            return qs.filter(
+                id__in=ops_learning_component_cache_response.used_ops_learning.values_list("appeal_document_id", flat=True)
+            )
+        return qs
+
 
 class FieldReportFilter(filters.FilterSet):
     dtype = filters.NumberFilter(field_name="dtype", lookup_expr="exact")

api/serializers.py

@@ -1553,11 +1553,14 @@ class Meta:
 
 
 class AppealDocumentAppealSerializer(serializers.ModelSerializer):
+    event = MiniEventSerializer(read_only=True)
+
     class Meta:
         model = Appeal
         fields = (
             "id",
             "code",
+            "event",
         )
 
 

main/permissions.py

@@ -20,6 +20,10 @@ class DenyGuestUserMutationPermission(permissions.BasePermission):
     """
 
     def _has_permission(self, request, view):
+        # Allow all safe methods (GET, HEAD, OPTIONS) which are non-mutating.
+        if request.method in permissions.SAFE_METHODS:
+            return True
+
         # For mutation methods (POST, PUT, DELETE, etc.):
         # Check if the user is authenticated.
         if not bool(request.user and request.user.is_authenticated):

main/urls.py

@@ -117,7 +117,6 @@
 router.register(r"public-per-stats", per_views.CountryPublicPerStatsViewset, basename="public_country_per_stats")
 router.register(r"per-stats", per_views.CountryPerStatsViewset, basename="country_per_stats")
 router.register(r"ops-learning", per_views.OpsLearningViewset, basename="ops_learning")
-router.register(r"ops-learning-summary", per_views.OpsLearningSummaryViewset, basename="ops_learning_summary")
 router.register(r"per-document-upload", per_views.PerDocumentUploadViewSet, basename="per_document_upload")
 
 router.register(r"personnel_deployment", deployment_views.PersonnelDeploymentViewset, basename="personnel_deployment")

per/cache.py

@@ -4,8 +4,13 @@
 
 import django_filters
 from django.core.serializers.json import DjangoJSONEncoder
+from django.db.models import Count, Prefetch
 
-from per.models import OpsLearningCacheResponse
+from per.models import (
+    OpsLearningCacheResponse,
+    OpsLearningComponentCacheResponse,
+    OpsLearningSectorCacheResponse,
+)
 
 
 class OpslearningSummaryCacheHelper:
@@ -47,15 +52,24 @@ def get_or_create(
         }
         hash_value = self.generate_hash(filter_data)
         # Check if the summary is already cached
-        ops_learning_summary, created = OpsLearningCacheResponse.objects.get_or_create(
+        # NOTE: count for the related components and sectors are prefetched
+        ops_learning_summary, created = OpsLearningCacheResponse.objects.prefetch_related(
+            "used_ops_learning",
+            Prefetch(
+                "ops_learning_component",
+                queryset=OpsLearningComponentCacheResponse.objects.select_related(
+                    "component",
+                ).annotate(count=Count("used_ops_learning")),
+            ),
+            Prefetch(
+                "ops_learning_sector",
+                queryset=OpsLearningSectorCacheResponse.objects.select_related(
+                    "sector",
+                ).annotate(count=Count("used_ops_learning")),
+            ),
+        ).get_or_create(
             used_filters_hash=hash_value,
             used_filters=filter_data,
-            status=OpsLearningCacheResponse.Status.SUCCESS,
             defaults={"status": OpsLearningCacheResponse.Status.PENDING},
         )
-        if not created:
-            return ops_learning_summary
-        # TODO send a http code of task is pending and return the task id
-        # transaction.on_commit(lambda: generate_summary.delay(ops_learning_summary, filter_data))
-        # return Response({"task_id": ops_learning_summary.id}, status=202)
-        return OpsLearningCacheResponse.objects.filter(status=OpsLearningCacheResponse.Status.SUCCESS).first()
+        return ops_learning_summary, filter_data

per/drf_views.py

@@ -2,6 +2,7 @@
 
 import pytz
 from django.conf import settings
+from django.db import transaction
 from django.db.models import Prefetch, Q
 from django.http import HttpResponse
 from django.shortcuts import get_object_or_404
@@ -34,6 +35,7 @@
     PerGeneralPermission,
     PerPermission,
 )
+from per.task import generate_summary
 from per.utils import filter_per_queryset_by_user_access
 
 from .admin_classes import RegionRestrictedAdmin
@@ -54,6 +56,8 @@
     NiceDocument,
     OpsLearning,
     OpsLearningCacheResponse,
+    OpsLearningComponentCacheResponse,
+    OpsLearningSectorCacheResponse,
     OrganizationTypes,
     Overview,
     PerAssessment,
@@ -678,6 +682,20 @@ class OpsLearningFilter(filters.FilterSet):
         widget=CSVWidget,
         queryset=FormComponent.objects.all(),
     )
+    insight_id = filters.NumberFilter(
+        label="Base Insight id for used extracts",
+        method="get_cache_response",
+    )
+    insight_sector_id = filters.NumberFilter(label="Sector insight id for used extracts", method="get_cache_response_sector")
+    insight_component_id = filters.NumberFilter(
+        label="Component insight id for used extracts",
+        method="get_cache_response_component",
+    )
+    # NOTE: overriding the fields for the typing issue
+    sector_validated = filters.NumberFilter(field_name="sector_validated", lookup_expr="exact")
+    per_component_validated = filters.NumberFilter(field_name="per_component_validated", lookup_expr="exact")
+    # NOTE: this field is used in summary generation
+    search_extracts = filters.CharFilter(method="get_filter_search_extracts")
 
     class Meta:
         model = OpsLearning
@@ -689,8 +707,6 @@ class Meta:
             "learning": ("exact", "icontains"),
             "learning_validated": ("exact", "icontains"),
             "organization_validated": ("exact",),
-            "sector_validated": ("exact",),
-            "per_component_validated": ("exact",),
             "appeal_code": ("exact", "in"),
             "appeal_code__code": ("exact", "icontains", "in"),
             "appeal_code__num_beneficiaries": ("exact", "gt", "gte", "lt", "lte"),
@@ -704,6 +720,31 @@ class Meta:
             "appeal_code__region": ("exact", "in"),
         }
 
+    def get_cache_response(self, queryset, name, value):
+        if value and (ops_learning_cache_response := OpsLearningCacheResponse.objects.filter(id=value).first()):
+            return queryset.filter(id__in=ops_learning_cache_response.used_ops_learning.all())
+        return queryset
+
+    def get_cache_response_sector(self, queryset, name, value):
+        if value and (ops_learning_sector_cache_response := OpsLearningSectorCacheResponse.objects.filter(id=value).first()):
+            return queryset.filter(id__in=ops_learning_sector_cache_response.used_ops_learning.all())
+        return queryset
+
+    def get_cache_response_component(self, queryset, name, value):
+        if value and (
+            ops_learning_component_cache_response := OpsLearningComponentCacheResponse.objects.filter(id=value).first()
+        ):
+            return queryset.filter(id__in=ops_learning_component_cache_response.used_ops_learning.all())
+        return queryset
+
+    def get_filter_search_extracts(self, queryset, name, value):
+        return queryset.filter(
+            Q(learning__icontains=value)
+            | Q(learning_validated__icontains=value)
+            | Q(appeal_code__name__icontains=value)
+            | Q(appeal_code__code__icontains=value)
+        )
+
 
 class OpsLearningViewset(viewsets.ModelViewSet):
     """
@@ -736,15 +777,27 @@ def get_queryset(self):
             return qs.select_related(
                 "appeal_code",
             ).prefetch_related(
-                "sector", "organization", "per_component", "sector_validated", "organization_validated", "per_component_validated"
+                "sector",
+                "organization",
+                "per_component",
+                "sector_validated",
+                "organization_validated",
+                "per_component_validated",
+                "appeal_code__event__countries_for_preview",
             )
         return (
             qs.filter(is_validated=True)
             .select_related(
                 "appeal_code",
             )
             .prefetch_related(
-                "sector", "organization", "per_component", "sector_validated", "organization_validated", "per_component_validated"
+                "sector",
+                "organization",
+                "per_component",
+                "sector_validated",
+                "organization_validated",
+                "per_component_validated",
+                "appeal_code__event__countries_for_preview",
             )
         )
 
@@ -813,7 +866,7 @@ def get_renderer_context(self):
     @extend_schema(
         request=None,
         filters=True,
-        responses=OpsLearningSummarySerializer(),
+        responses=OpsLearningSummarySerializer,
     )
     @action(
         detail=False,
@@ -825,7 +878,11 @@ def summary(self, request):
         """
         Get the Ops Learning Summary based on the filters
         """
-        ops_learning_summary_instance = OpslearningSummaryCacheHelper.get_or_create(request, [self.filterset_class])
+        ops_learning_summary_instance, filter_data = OpslearningSummaryCacheHelper.get_or_create(request, [self.filterset_class])
+        if ops_learning_summary_instance.status == OpsLearningCacheResponse.Status.SUCCESS:
+            return response.Response(OpsLearningSummarySerializer(ops_learning_summary_instance).data)
+
+        transaction.on_commit(lambda: generate_summary.delay(ops_learning_summary_instance.id, filter_data))
         return response.Response(OpsLearningSummarySerializer(ops_learning_summary_instance).data)
 
 
@@ -839,10 +896,3 @@ def get_queryset(self):
         queryset = super().get_queryset()
         user = self.request.user
         return filter_per_queryset_by_user_access(user, queryset)
-
-
-class OpsLearningSummaryViewset(viewsets.ReadOnlyModelViewSet):
-    queryset = OpsLearningCacheResponse.objects.all()
-    serializer_class = OpsLearningSummarySerializer
-    permission_classes = [permissions.IsAuthenticated]
-    pagination_class = None

per/factories.py

@@ -64,7 +64,7 @@ class Meta:
 
 class FormComponentFactory(factory.django.DjangoModelFactory):
     area = factory.SubFactory(FormAreaFactory)
-    title = fuzzy.FuzzyText(length=50, prefix="component-")
+    title = factory.Faker("sentence", nb_words=5)
 
     class Meta:
         model = FormComponent
@@ -114,20 +114,20 @@ class Meta:
 
 class OpsLearningCacheResponseFactory(factory.django.DjangoModelFactory):
     used_filters_hash = fuzzy.FuzzyText(length=20)
-    insights1_title = fuzzy.FuzzyText(length=50, prefix="insights1-title-")
-    insights1_content = fuzzy.FuzzyText(length=100, prefix="insights1-content-")
-    insights2_title = fuzzy.FuzzyText(length=50, prefix="insights2-title-")
-    insights2_content = fuzzy.FuzzyText(length=100, prefix="insights2-content-")
-    insights3_title = fuzzy.FuzzyText(length=50, prefix="insights3-title-")
-    insights3_content = fuzzy.FuzzyText(length=100, prefix="insights3-content-")
+    insights1_title = factory.Faker("sentence", nb_words=5)
+    insights1_content = factory.Faker("sentence", nb_words=20)
+    insights2_title = factory.Faker("sentence", nb_words=5)
+    insights2_content = factory.Faker("sentence", nb_words=25)
+    insights3_title = factory.Faker("sentence", nb_words=10)
+    insights3_content = factory.Faker("sentence", nb_words=30)
 
     class Meta:
         model = OpsLearningCacheResponse
 
 
 class OpsLearningSectorCacheResponseFactory(factory.django.DjangoModelFactory):
     filter_response = factory.SubFactory(OpsLearningCacheResponseFactory)
-    content = fuzzy.FuzzyText(length=50)
+    content = factory.Faker("sentence", nb_words=30)
     sector = factory.SubFactory(SectorTagFactory)
 
     class Meta:
@@ -136,7 +136,7 @@ class Meta:
 
 class OpsLearningComponentCacheResponseFactory(factory.django.DjangoModelFactory):
     filter_response = factory.SubFactory(OpsLearningCacheResponseFactory)
-    content = fuzzy.FuzzyText(length=50)
+    content = factory.Faker("sentence", nb_words=30)
     component = factory.SubFactory(FormComponentFactory)
 
     class Meta:

per/management/commands/create_dummy_opslearningsummary.py

@@ -8,7 +8,7 @@
     OpsLearningFactory,
     OpsLearningSectorCacheResponseFactory,
 )
-from per.models import OpsLearningCacheResponse
+from per.models import OpsLearning, OpsLearningCacheResponse
 
 
 class Command(BaseCommand):
@@ -35,7 +35,10 @@ def generate_component_response(self, ops_learnings: list, ops_learning_cache_re
             ops_learning_component_cache.used_ops_learning.add(*ops_learnings)
 
     def generate_ops_learning_summary(self):
-        selected_ops_learning = OpsLearningFactory.create_batch(50, is_validated=True)
+        selected_ops_learning = OpsLearning.objects.filter(is_validated=True)[:50]
+        if not selected_ops_learning:
+            selected_ops_learning = OpsLearningFactory.create_batch(50, is_validated=True)
+            self.stderr.write(self.style.ERROR("No OpsLearning data found. Create Fake OpsLearning data first."))
 
         # Generating dummy OpsLearningCacheResponse
         dummy_ops_learning_cache_responses = OpsLearningCacheResponseFactory.create_batch(