Skip to content

Commit

Permalink
Fix duplicate stats generation in country page
Browse files Browse the repository at this point in the history
  • Loading branch information
@k9845
k9845 committed Jun 20, 2024
1 parent b26019f commit b61b153
Show file tree
Hide file tree
Showing 5 changed files with 133 additions and 109 deletions.
108 changes: 58 additions & 50 deletions api/drf_views.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -32,7 +32,7 @@
from api.filter_set import (
Admin2Filter,
AppealDocumentFilter,
AppealHistoryFilter,
AppealFilter,
CountryFilter,
CountryFilterRMD,
CountryKeyDocumentFilter,
Expand Down Expand Up @@ -66,7 +66,6 @@
Admin2,
Appeal,
AppealDocument,
AppealHistory,
AppealType,
Country,
CountryKeyDocument,
Expand Down Expand Up @@ -97,8 +96,8 @@
Admin2Serializer,
AppealDocumentSerializer,
AppealDocumentTableauSerializer,
AppealHistorySerializer,
AppealHistoryTableauSerializer,
AppealSerializer,
AppealTableauSerializer,
CountryDisasterTypeCountSerializer,
CountryDisasterTypeMonthlySerializer,
CountryGeoSerializer,
Expand Down Expand Up @@ -262,49 +261,68 @@ def get_databank(self, request, pk):
)
def get_country_figure(self, request, pk):
country = self.get_object()
end_date = timezone.now()
start_date = end_date + timedelta(days=-2 * 365)
start_date = request.GET.get("start_date", start_date)
end_date = request.GET.get("end_date", end_date)
appeal_conditions = Q(atype=AppealType.APPEAL)

all_appealhistory = AppealHistory.objects.select_related("appeal").filter(appeal__code__isnull=False)
all_appealhistory = all_appealhistory.filter(country=country)
start_date_to = timezone.now()
start_date_from = start_date_to + timedelta(days=-2 * 365)
start_date = request.GET.get("start_date_from", start_date_from)
end_date = request.GET.get("start_date_to", start_date_to)
appeal_conditions = Q(atype=AppealType.APPEAL) | Q(atype=AppealType.INTL)

all_appeal = Appeal.objects.select_related("country").filter(
code__isnull=False,
)
all_appeal = all_appeal.filter(country=country)
if start_date and end_date:
all_appealhistory = all_appealhistory.filter(start_date__lte=end_date, end_date__gte=start_date)
appeals_aggregated = all_appealhistory.annotate(
all_appeal = all_appeal.filter(
start_date__lte=end_date,
start_date__gte=start_date,
)
appeals_aggregated = all_appeal.annotate(
appeal_with_dref=Count(
Case(
When(Q(atype=AppealType.DREF), then=1),
output_field=IntegerField(),
)
),
appeal_without_dref=Count(Case(When(Q(atype=AppealType.APPEAL), then=1), output_field=IntegerField()), distinct=True),
appeal_without_dref=Count(Case(When(appeal_conditions, then=1), output_field=IntegerField())),
total_population=(
Case(
When(appeal_conditions | Q(atype=AppealType.DREF), then=F("num_beneficiaries")),
output_field=IntegerField(),
)
),
amount_requested_all=(
Case(
When(appeal_conditions, then=F("amount_requested")),
output_field=IntegerField(),
)
),
amordref=(
Case(
When(appeal_conditions | Q(atype=AppealType.DREF), then=F("amount_requested")),
output_field=IntegerField(),
)
),
amount_funded_all=(
amof=(
Case(
When(appeal_conditions, then=F("amount_funded")),
output_field=IntegerField(),
)
),
amofdref=(
Case(
When(appeal_conditions | Q(atype=AppealType.DREF), then=F("amount_funded")),
output_field=IntegerField(),
)
),
emergencies_count=Count(F("appeal__event_id"), distinct=True),
emergencies_count=Count(F("event_id"), distinct=True),
).aggregate(
active_drefs=Sum("appeal_with_dref"),
active_appeals=Sum("appeal_without_dref"),
target_population=Sum("total_population"),
total_amount_requested=Sum("amount_requested_all"),
total_amount_funded=Sum("amount_funded_all"),
amount_requested=Sum("amount_requested_all"),
amount_requested_dref_included=Sum("amordref"),
amount_funded=Sum("amof"),
amount_funded_dref_included=Sum("amofdref"),
emergencies=Sum("emergencies_count"),
)
return Response(CountryKeyFigureSerializer(appeals_aggregated).data)
Expand All @@ -318,10 +336,10 @@ def get_country_figure(self, request, pk):
@action(detail=True, url_path="disaster-count", pagination_class=None)
def get_country_disaster_count(self, request, pk):
country = self.get_object()
end_date = timezone.now()
start_date = end_date + timedelta(days=-2 * 365)
start_date = request.GET.get("start_date", start_date)
end_date = request.GET.get("end_date", end_date)
start_date_to = timezone.now()
start_date_from = start_date_to + timedelta(days=-2 * 365)
start_date = request.GET.get("start_date_from", start_date_from)
end_date = request.GET.get("start_date_to", start_date_to)

queryset = (
Event.objects.filter(
Expand Down Expand Up @@ -349,16 +367,16 @@ def get_country_disaster_count(self, request, pk):
@action(detail=True, url_path="disaster-monthly-count", pagination_class=None)
def get_country_disaster_monthly_count(self, request, pk):
country = self.get_object()
end_date = timezone.now()
start_date = end_date + timedelta(days=-2 * 365)
start_date = request.GET.get("start_date", start_date)
end_date = request.GET.get("end_date", end_date)
start_date_to = timezone.now()
start_date_from = start_date_to + timedelta(days=-2 * 365)
start_date = request.GET.get("start_date_from", start_date_from)
end_date = request.GET.get("start_date_to", start_date_to)
queryset = (
Event.objects.filter(
countries__in=[country.id],
dtype__isnull=False,
)
.annotate(date=TruncMonth("created_at"))
.annotate(date=TruncMonth("disaster_start_date"))
.values("date", "countries", "dtype")
.annotate(
appeal_targeted_population=Coalesce(
Expand Down Expand Up @@ -404,18 +422,18 @@ def get_country_disaster_monthly_count(self, request, pk):
@action(detail=True, url_path="historical-disaster", pagination_class=None)
def get_country_historical_disaster(self, request, pk):
country = self.get_object()
end_date = timezone.now()
start_date = end_date + timedelta(days=-2 * 365)
start_date = request.GET.get("start_date", start_date)
end_date = request.GET.get("end_date", end_date)
start_date_to = timezone.now()
start_date_from = start_date_to + timedelta(days=-2 * 365)
start_date = request.GET.get("start_date_from", start_date_from)
end_date = request.GET.get("start_date_to", start_date_to)
dtype = request.GET.get("dtype", None)

queryset = (
Event.objects.filter(
countries__in=[country.id],
dtype__isnull=False,
)
.annotate(date=TruncMonth("created_at"))
.annotate(date=TruncMonth("disaster_start_date"))
.values("date", "dtype", "countries")
.annotate(
appeal_targeted_population=Coalesce(
Expand Down Expand Up @@ -744,27 +762,19 @@ def get_serializer_class(self):
class AppealViewset(mixins.ListModelMixin, viewsets.GenericViewSet):
"""Used to get Appeals from AppealHistory. Has no 'read' option, just 'list'."""

# queryset = Appeal.objects.select_related('dtype', 'country', 'region').all()
# queryset = AppealHistory.objects.select_related('appeal__event', 'dtype', 'country', 'region').all()
queryset = AppealHistory.objects.select_related("appeal__event", "dtype", "country", "region").filter(
appeal__code__isnull=False
)
# serializer_class = AppealSerializer
serializer_class = AppealHistorySerializer
queryset = Appeal.objects.select_related("event", "dtype", "country", "region").filter(code__isnull=False)
serializer_class = AppealSerializer
ordering_fields = "__all__"
# filterset_class = AppealFilter
filterset_class = AppealHistoryFilter
filterset_class = AppealFilter
search_fields = (
"appeal__name",
"name",
"code",
) # for /docs

def get_serializer_class(self):
if is_tableau(self.request) is True:
return AppealHistoryTableauSerializer
# return AppealTableauSerializer
return AppealHistorySerializer
# return AppealSerializer
return AppealTableauSerializer
return AppealSerializer

def remove_unconfirmed_event(self, obj):
if obj["needs_confirmation"]:
Expand All @@ -776,9 +786,7 @@ def remove_unconfirmed_events(self, objs):

# Overwrite to exclude the events which require confirmation
def list(self, request, *args, **kwargs):
now = timezone.now()
date = request.GET.get("date", now)
queryset = self.filter_queryset(self.get_queryset()).filter(valid_from__lt=date, valid_to__gt=date)
queryset = self.filter_queryset(self.get_queryset())

page = self.paginate_queryset(queryset)
if page is not None:
Expand Down
24 changes: 24 additions & 0 deletions api/filter_set.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -204,14 +204,38 @@ class AppealFilter(filters.FilterSet):
code = filters.CharFilter(field_name="code", lookup_expr="exact")
status = filters.NumberFilter(field_name="status", lookup_expr="exact")
id = filters.NumberFilter(field_name="id", lookup_expr="exact")
appeal_id = filters.NumberFilter(
field_name="appeal_id", lookup_expr="exact", help_text="Use this (or code) for appeal identification."
)
district = filters.ModelMultipleChoiceFilter(
field_name="country__district", queryset=District.objects.all(), label="district", method="get_country_district"
)
admin2 = filters.ModelMultipleChoiceFilter(
field_name="country__district__admin2",
queryset=Admin2.objects.all(),
label="admin2",
method="get_country_admin2",
)

class Meta:
model = Appeal
fields = {
"start_date": ("exact", "gt", "gte", "lt", "lte"),
"end_date": ("exact", "gt", "gte", "lt", "lte"),
"real_data_update": ("exact", "gt", "gte", "lt", "lte"),
"country__iso3": ("exact",),
}

def get_country_district(self, qs, name, value):
if value:
return qs.filter(country__district=value).distinct()
return qs

def get_country_admin2(self, qs, name, value):
if value:
return qs.filter(country__district__admin2=value).distinct()
return qs


class AppealHistoryFilter(filters.FilterSet):
atype = filters.NumberFilter(field_name="atype", lookup_expr="exact")
Expand Down
11 changes: 7 additions & 4 deletions api/serializers.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2118,6 +2118,7 @@ class AggregateHeaderFiguresSerializer(serializers.Serializer):
amount_requested = serializers.IntegerField()
amount_requested_dref_included = serializers.IntegerField()
amount_funded = serializers.IntegerField()
amount_funded_dref_included = serializers.IntegerField()


# SearchPage Serializer
Expand Down Expand Up @@ -2292,8 +2293,8 @@ class AggregateByDtypeSerializer(serializers.Serializer):


class CountryKeyFigureInputSerializer(serializers.Serializer):
start_date = serializers.DateField(required=False)
end_date = serializers.DateField(required=False)
start_date_from = serializers.DateField(required=False)
start_date_to = serializers.DateField(required=False)
dtype = serializers.IntegerField(required=False)


Expand All @@ -2304,9 +2305,11 @@ class CountryKeyClimateInputSerializer(serializers.Serializer):
class CountryKeyFigureSerializer(serializers.Serializer):
active_drefs = serializers.IntegerField()
active_appeals = serializers.IntegerField()
amount_requested = serializers.IntegerField()
target_population = serializers.IntegerField()
total_amount_requested = serializers.IntegerField()
total_amount_funded = serializers.IntegerField()
amount_requested_dref_included = serializers.IntegerField()
amount_funded = serializers.IntegerField()
amount_funded_dref_included = serializers.IntegerField()
emergencies = serializers.IntegerField()


Expand Down
64 changes: 29 additions & 35 deletions api/test_views.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,6 @@
import api.models as models
from api.factories.event import (
AppealFactory,
AppealHistoryFactory,
AppealType,
EventFactory,
EventFeaturedDocumentFactory,
Expand Down Expand Up @@ -543,58 +542,53 @@ def test_appeal_key_figure(self):
)
event2 = EventFactory.create(name="test0", dtype=dtype1, num_affected=10000, countries=[country1])
event3 = EventFactory.create(name="test2", dtype=dtype2, num_affected=99999, countries=[country2])
appeal1 = AppealFactory.create(
event=event1, dtype=dtype1, num_beneficiaries=9000, amount_requested=10000, amount_funded=1899999, code=12
)
appeal2 = AppealFactory.create(
event=event2, dtype=dtype2, num_beneficiaries=90023, amount_requested=100440, amount_funded=12299999, code=123
)
appeal3 = AppealFactory.create(
event=event3, dtype=dtype2, num_beneficiaries=91000, amount_requested=10000888, amount_funded=678888, code=1234
)
AppealHistoryFactory.create(
appeal=appeal1,
AppealFactory.create(
event=event1,
dtype=dtype1,
num_beneficiaries=9000,
amount_requested=10000,
amount_funded=1899999,
country=country1,
atype=AppealType.APPEAL,
code=12,
start_date="2024-1-1",
end_date="2024-1-1",
atype=AppealType.APPEAL,
country=country1,
)
AppealHistoryFactory.create(
appeal=appeal2,
AppealFactory.create(
event=event2,
dtype=dtype2,
num_beneficiaries=1,
amount_requested=1,
amount_funded=1,
country=country1,
atype=AppealType.DREF,
num_beneficiaries=90023,
amount_requested=100440,
amount_funded=12299999,
code=123,
start_date="2024-2-2",
end_date="2024-2-2",
atype=AppealType.DREF,
country=country1,
)
AppealHistoryFactory.create(
appeal=appeal3,
AppealFactory.create(
event=event3,
dtype=dtype2,
num_beneficiaries=1,
amount_requested=1,
amount_funded=1,
country=country1,
atype=AppealType.APPEAL,
num_beneficiaries=91000,
amount_requested=10000888,
amount_funded=678888,
code=1234,
start_date="2024-3-3",
end_date="2024-3-3",
atype=AppealType.APPEAL,
country=country1,
)
AppealHistoryFactory.create(
appeal=appeal3,
AppealFactory.create(
event=event3,
dtype=dtype2,
num_beneficiaries=1,
amount_requested=1,
amount_funded=1,
country=country1,
atype=AppealType.APPEAL,
num_beneficiaries=91000,
amount_requested=10000888,
amount_funded=678888,
code=12345,
start_date="2024-4-4",
end_date="2024-4-4",
atype=AppealType.APPEAL,
country=country1,
)
url = f"/api/v2/country/{country1.id}/figure/"
self.client.force_authenticate(self.user)
Expand Down
Loading

0 comments on commit b61b153

Please sign in to comment.