Caching implementation and add changes on parsing

per/cache.py

@@ -50,14 +50,12 @@ def get_or_create(
         ops_learning_summary, created = OpsLearningCacheResponse.objects.get_or_create(
             used_filters_hash=hash_value,
             used_filters=filter_data,
+            status=OpsLearningCacheResponse.Status.SUCCESS,
             defaults={"status": OpsLearningCacheResponse.Status.PENDING},
         )
-        if not created and ops_learning_summary.status == OpsLearningCacheResponse.Status.SUCCESS:
+        if not created:
             return ops_learning_summary
-        # TODO: Create a new summary and cache it
         # TODO send a http code of task is pending and return the task id
-        # return transaction.on_commit(lambda: generate_summary.delay(ops_learning_summary, filter_data))
-        # return OpsLearningCacheResponse.objects.filter(status=OpsLearningCacheResponse.Status.SUCCESS).first()
-        from per.task import generate_summary
-
-        return generate_summary(ops_learning_summary, filter_data)
+        # transaction.on_commit(lambda: generate_summary.delay(ops_learning_summary, filter_data))
+        # return Response({"task_id": ops_learning_summary.id}, status=202)
+        return OpsLearningCacheResponse.objects.filter(status=OpsLearningCacheResponse.Status.SUCCESS).first()

per/ops_learning_summary.py

@@ -113,6 +113,12 @@ def count_tokens(string, encoding_name):
         encoding = tiktoken.get_encoding(encoding_name)
         return len(encoding.encode(string))
 
+    @staticmethod
+    def change_ops_learning_status(instance: OpsLearningCacheResponse, status: OpsLearningCacheResponse.Status):
+        """Changes the status of the OPS learning instance."""
+        instance.status = status
+        instance.save(update_fields=["status"])
+
     @classmethod
     def fetch_ops_learnings(self, filter_data):
         """Fetches the OPS learnings from the database."""
@@ -607,10 +613,10 @@ def _modify_summary(summary: dict) -> dict:
             Checks if the "Confidence level" is present in the primary response and skipping for the secondary summary
             """
             for key, value in summary.items():
-                if key == "contradictory reports":
+                if key == "contradictory reports" or "confidence level" in value:
                     continue
                 if "Confidence level" in value["content"]:
-                    confidence_value = value["content"].split("Confidence level:")[-1]
+                    confidence_value = value["content"].split("Confidence level:")[-1].strip()
                     value["content"] = value["content"].split("Confidence level:")[0]
                     value["confidence level"] = confidence_value
 
@@ -703,7 +709,7 @@ def save_to_db(
                 )
             else:
                 logger.error(f"Invalid type '{type}' on secondary summary.")
-
+        self.change_ops_learning_status(ops_learning_summary_instance, OpsLearningCacheResponse.Status.SUCCESS)
         logger.info("Saved to database.")
 
     @classmethod

per/task.py

@@ -1,17 +1,30 @@
 from celery import shared_task
 
+from api.logger import logger
 from per.models import OpsLearningCacheResponse
 from per.ops_learning_summary import OpsLearningSummaryTask
 
 
 @shared_task
 def generate_summary(ops_learning_summary_instance: OpsLearningCacheResponse, filter_data: dict):
-    regional_list, global_list, country_list = OpsLearningSummaryTask.generate_priotization_list()
-    prioritized_learnings = OpsLearningSummaryTask.prioritize_components(filter_data, regional_list, global_list, country_list)
-    primary_learning_df, secondary_learning_df = OpsLearningSummaryTask.prioritize_excerpts(prioritized_learnings)
-    primary_learning_prompt, secondary_learning_prompt = OpsLearningSummaryTask.format_prompt(
-        primary_learning_df, secondary_learning_df, filter_data
-    )
-    OpsLearningSummaryTask.get_or_create_summary(
-        ops_learning_summary_instance, primary_learning_prompt, secondary_learning_prompt
-    )
+    try:
+        OpsLearningSummaryTask.change_ops_learning_status(
+            instance=ops_learning_summary_instance, status=OpsLearningCacheResponse.Status.STARTED
+        )
+        regional_list, global_list, country_list = OpsLearningSummaryTask.generate_priotization_list()
+        prioritized_learnings = OpsLearningSummaryTask.prioritize_components(
+            filter_data, regional_list, global_list, country_list
+        )
+        primary_learning_df, secondary_learning_df = OpsLearningSummaryTask.prioritize_excerpts(prioritized_learnings)
+        primary_learning_prompt, secondary_learning_prompt = OpsLearningSummaryTask.format_prompt(
+            primary_learning_df, secondary_learning_df, filter_data
+        )
+        OpsLearningSummaryTask.get_or_create_summary(
+            ops_learning_summary_instance, primary_learning_prompt, secondary_learning_prompt
+        )
+    except Exception as e:
+        OpsLearningSummaryTask.change_ops_learning_status(
+            instance=ops_learning_summary_instance, status=OpsLearningCacheResponse.Status.FAILED
+        )
+        logger.error(e)
+        raise e