From 45779341f5d92aa038a40598dc12094125b9bc0f Mon Sep 17 00:00:00 2001 From: Alfie Liu <109685890+Alfiejz@users.noreply.github.com> Date: Tue, 11 Jul 2023 18:43:20 +0100 Subject: [PATCH] Alert refactor merge (#29) * New feed facade and celery task improvements * Update main_cap-aggregator(dev).yml * added separate configurations for each cap source format * added region data * reset migrations * removed region usage before declaration * fixed variation scope issue * created migration * injecting regions * injecting countries (#7) * matched CAP alerts to country polygons * Integrated Celery and RabbitMQ * Preparation for region data injection (#6) * Update main_cap-aggregator(dev).yml * added separate configurations for each cap source format * added region data * reset migrations * removed region usage before declaration * fixed variation scope issue * Add or update the Azure App Service build and deployment workflow config * add celery tasks --------- Co-authored-by: HangZhou01 * added celery startup commands * changed startup.sh * added filter for unexpired alerts, fixed cors policy * enabled github workflow tests for all branches, whitenoise package version bump * fixed cors policy origin path * disabled celery for deployment * reverted cors policy * New feed facade and celery task improvements * Add or update the Azure App Service build and deployment workflow config * add celery tasks * merge the develop branch * Polling Feeds at Different Rate * Automatically Polling Alerts when a New Feed is created * Automatically Deleting or Updating Task when there is a Feed gets deleted or updated * Fix bugs on Automatically Deleting Updating Task when there is a Feed gets deleted or updated * fixed minor bugs * refactored celery code, fixed lots of bugs --------- Co-authored-by: HangZhou01 * fixed celery task name issue * fixed naming issue * resolved conflicts between develop and main * fixed merge issue --------- Co-authored-by: HangZhou01 * New country geodata, feed facade redesign, graphql filters * Update main_cap-aggregator(dev).yml * added separate configurations for each cap source format * added region data * reset migrations * removed region usage before declaration * fixed variation scope issue * created migration * injecting regions * injecting countries (#7) * matched CAP alerts to country polygons * Integrated Celery and RabbitMQ * Preparation for region data injection (#6) * Update main_cap-aggregator(dev).yml * added separate configurations for each cap source format * added region data * reset migrations * removed region usage before declaration * fixed variation scope issue * Add or update the Azure App Service build and deployment workflow config * add celery tasks --------- Co-authored-by: HangZhou01 * added celery startup commands * changed startup.sh * added filter for unexpired alerts, fixed cors policy * enabled github workflow tests for all branches, whitenoise package version bump * fixed cors policy origin path * disabled celery for deployment * reverted cors policy * New feed facade and celery task improvements * Add or update the Azure App Service build and deployment workflow config * add celery tasks * merge the develop branch * Polling Feeds at Different Rate * Automatically Polling Alerts when a New Feed is created * Automatically Deleting or Updating Task when there is a Feed gets deleted or updated * Fix bugs on Automatically Deleting Updating Task when there is a Feed gets deleted or updated * fixed minor bugs * refactored celery code, fixed lots of bugs --------- Co-authored-by: HangZhou01 * fixed celery task name issue * fixed naming issue * resolved conflicts between develop and main * fixed merge issue * Added continents, changed country polygon data source * New feed facade and celery task improvements * Update main_cap-aggregator(dev).yml * added separate configurations for each cap source format * added region data * reset migrations * removed region usage before declaration * fixed variation scope issue * created migration * injecting regions * injecting countries (#7) * matched CAP alerts to country polygons * Integrated Celery and RabbitMQ * Preparation for region data injection (#6) * Update main_cap-aggregator(dev).yml * added separate configurations for each cap source format * added region data * reset migrations * removed region usage before declaration * fixed variation scope issue * Add or update the Azure App Service build and deployment workflow config * add celery tasks --------- Co-authored-by: HangZhou01 * added celery startup commands * changed startup.sh * added filter for unexpired alerts, fixed cors policy * enabled github workflow tests for all branches, whitenoise package version bump * fixed cors policy origin path * disabled celery for deployment * reverted cors policy * New feed facade and celery task improvements * Add or update the Azure App Service build and deployment workflow config * add celery tasks * merge the develop branch * Polling Feeds at Different Rate * Automatically Polling Alerts when a New Feed is created * Automatically Deleting or Updating Task when there is a Feed gets deleted or updated * Fix bugs on Automatically Deleting Updating Task when there is a Feed gets deleted or updated * fixed minor bugs * refactored celery code, fixed lots of bugs --------- Co-authored-by: HangZhou01 * fixed celery task name issue * fixed naming issue * resolved conflicts between develop and main * fixed merge issue --------- Co-authored-by: HangZhou01 * added continents, changed country polygon data source * merge continents --------- Co-authored-by: HangZhou01 * fixed merge duplication * fixed inconsistencies * -enabled geo injection * fixed graphql schema * deployment fix: inject continents only * deployment fix: changed celery task name * deployment fix: inject continents * deployment fix: removed save to database * deployment fix: try except inject continents * deployment fix: try except error message * deployment fix: try except print error * deployment fix: changed file path * deployment fix: changed file path * deployment fix: replaced filename dashes with underscores * deployment fix: changed relative paths * multipolygons added for countries * improved admin fields, filters, and displays, added cors, fixed celery tasks bug, fixed duplicate iso3 sources, updated settings.py celery format * added description, senderName, source CAP alert fields, added more sources, standardised [long, lat] coordinates, added more admin filters, displays * added graphql filters for alerts and countries --------- Co-authored-by: HangZhou01 * Update README.md Fixed documentation mistake * Alert model changed to comply with CAP 1.2, changed format structure to be more modular to allow for complex alert parsing * added meteoalarm, aws, nws_us alert feed advanced formats * fixed broken tests from alert model change * fixed feed format NoneType errors, tidied feed facade * updated graphql for new Alert and AlertInfo structures * revised README.md * removed old file * fixed merge issues --------- Co-authored-by: HangZhou01 --- README.md | 16 +- cap_feed/admin.py | 24 +- cap_feed/alert_processing.py | 224 ----------------- cap_feed/alert_processor.py | 19 ++ cap_feed/data_injector.py | 121 +++++++++ cap_feed/formats/__init__.py | 0 cap_feed/formats/aws.py | 64 +++++ cap_feed/formats/format_handler.py | 16 ++ cap_feed/formats/meteoalarm.py | 63 +++++ cap_feed/formats/nws_us.py | 67 +++++ cap_feed/formats/utils.py | 8 + ...t_identifier_alter_alert_scope_and_more.py | 33 +++ ...ea_desc_remove_alert_certainty_and_more.py | 152 ++++++++++++ cap_feed/models.py | 231 ++++++++++++------ cap_feed/tasks.py | 2 +- cap_feed/templates/cap_feed/index.html | 20 +- cap_feed/tests.py | 85 +++++-- cap_feed/views.py | 24 +- capaggregator/schema.py | 20 +- capaggregator/settings.py | 2 +- 20 files changed, 830 insertions(+), 361 deletions(-) delete mode 100644 cap_feed/alert_processing.py create mode 100644 cap_feed/alert_processor.py create mode 100644 cap_feed/data_injector.py create mode 100644 cap_feed/formats/__init__.py create mode 100644 cap_feed/formats/aws.py create mode 100644 cap_feed/formats/format_handler.py create mode 100644 cap_feed/formats/meteoalarm.py create mode 100644 cap_feed/formats/nws_us.py create mode 100644 cap_feed/formats/utils.py create mode 100644 cap_feed/migrations/0015_alter_alert_identifier_alter_alert_scope_and_more.py create mode 100644 cap_feed/migrations/0016_remove_alert_area_desc_remove_alert_certainty_and_more.py diff --git a/README.md b/README.md index d670e8b..d9e60cd 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,6 @@ # IFRC/UCL Alert Hub - CAP Aggregator -The CAP Aggregator is an alert aggregation service built for IFRC's Alert Hub. Public alerts use the Common Alerting Protocol (CAP) international standard. +The CAP Aggregator is an alert aggregation service built for IFRC's Alert Hub. Public alerts use the Common Alerting Protocol (CAP) Version 1.2 standard. This is a Python web app using the Django framework and the Azure Database for PostgreSQL relational database service. The Django app is hosted in a fully managed Azure App Service. Requests to hundreds of publicly available alert sources are managed by Celery and Redis. Aggregated alerts are then made available to the Alert Hub via a GraphQL API. @@ -37,26 +37,24 @@ The allocation of countries into regions and continents is necessary for easier ## Alert Aggregation Process *Alerts are retrieved and processed before they are handed off to be displayed on the Alert Hub and alert subscription system.* -New alert sources are added from the Feed Facade and polling intervals are used to adjust the frequency of requests to the alerting source. Alert sources with the same polling interval are automatically grouped into the same periodic task used by the *Celery Beat* scheduler. These tasks are handed off to *Redis* and *Celery* workers for asynchronous background processing. +New alert sources are added by an admin from the Feed Facade and polling intervals are used to adjust the frequency of requests to the alerting source. Alert sources with the same polling interval are automatically grouped into the same periodic task used by the *Celery Beat* scheduler. These tasks are handed off to *Redis* and *Celery* workers for asynchronous background processing. When processing the CAP feed of alerting sources, a processing template or format is used to interpret the different xml formats. For example, Meteoalarm represents a network of public weather services across Europe, and these European countries encode their cap alerts in the same xml format. The 'meteoalarm' format can therefore be selected when adding MeteoAlarm alerting sources in the feed facade, but a different format would need to be used to interpret alerts from the Algerian Meteorological Office. -Formats are very convenient for admin users but have an inevitable drawback — they have to be created manually by developers and updated if alerting sources make changes to their CAP alerts. The number of available formats is currently limited, but we intend to greatly expand on this number as we complete other prioritised tasks such as work relating to *Django Channels*. +Formats are very convenient for admin users and can guarantee alerts are processed correctly. But they inevitably have to be manually created by developers and updated if alerting sources make changes to their alert feed format. -CAP alerts including all kinds of metadata specific to each format is interpreted and saved to the Postgresql database. Dates and times are standardised across the system using the UTC timezone. Some alerting sources keep outdated alerts on their alert feeds, so expired alerts are identified and are not saved. +The CAP-aggregator processes alerts according to the CAP-V1.2 specification which details alert elements and sub-elements such as *info*. Dates and times are standardised across the system using the UTC timezone. Some alerting sources keep outdated alerts on their alert feeds, so expired alerts are identified and are not saved. -Another periodic task for removing expired alerts also runs continously in the background. This task is responsible for identifying and removing alerts which have expired since being saved to the database. +Another periodic task for removing expired alerts also runs continously in the background. This task is responsible for identifying and removing alerts which have expired since being saved to the database. However, the alert expiry date and time is contained in the *info* element according to CAP V-1.2. Therefore it is theoretically possible for multiple *info* elements to have different expiry times. Expired *info* elements are automatically removed, and the *alert* element (the actual alert itself) will be removed if all *info* elements have expired or been removed. Alerts are aggregated by countries, regions, and continents. Using filtered queries with GraphQL, the Alert Hub and other broadcasters can easily fetch only the relevant alerts, reducing unnecessary strain on the system. -> During development, we have discovered that a significant number of alerts are issued but then retracted by alerting sources before the stated expiry time. This may be a problem for alert subscriptions and we are currently discussing on our strategy for this issue. - ## Feed Facade *Admin users can manage countries, regions, continents, sources, and each individual alert using the Feed Facade.* Each alerting source and their alerts belong to a country, and each country belongs to a particular region and continent. Therefore, it is necessary for regions and continents to exist first before countries can be added (although all regions and continents already exist in the system). Similarly, a new country needs be created by an admin user before a new alerting source can be added for that country. -Deleting a region or continent would delete all countries belonging to them. In a chain reaction, all alerts and sources belonging to the deleted countries would also be deleted. This current behaviour is clearly unsafe and possibly undesirable but is convenient for development. This can be changed after some discussion. Lastly, deleting an alerting source does not delete existing alerts from the same country. This is because alerts belong to the country of the alerting source, but not to the sources themselves. +Deleting a region or continent would delete all countries belonging to them. In a chain reaction, all alerts and sources belonging to the deleted countries would also be deleted. This current behaviour is possibly unsafe and undesirable but is convenient for development. Lastly, deleting an alerting source also deletes existing alerts from the same country. Search functions, filters and sortable columns are available when they would be relevant. For example, an admin user could filter sources by format (e.g., meteoalarm) or search for sources belonging to a particular country using the search bar on the same page. @@ -83,4 +81,4 @@ Start celery worker and sceduler for local development: ``` celery -A capaggregator worker --pool=solo -l info celery -A capaggregator beat -l info -``` \ No newline at end of file +``` diff --git a/cap_feed/admin.py b/cap_feed/admin.py index e1f5558..f311668 100644 --- a/cap_feed/admin.py +++ b/cap_feed/admin.py @@ -1,13 +1,30 @@ from django.contrib import admin -from .models import Alert, Continent, Region, Country, Source +from .models import Alert, AlertInfo, Continent, Region, Country, Source from django_celery_beat.models import CrontabSchedule, ClockedSchedule, SolarSchedule, IntervalSchedule from django_celery_results.models import GroupResult +class AlertInfoAdmin(admin.ModelAdmin): + list_display = ["alert", "language"] + list_filter = ["alert__source_feed"] + fieldsets = [ + ("Administration", {"fields": ["alert"]}), + ("Alert Info" , {"fields": ["language", "category", "event", "response_type", "urgency", "severity", "certainty", "audience", "event_code", "effective", "onset", "expires", "sender_name", "headline", "description", "instruction", "web", "contact", "parameter"]}), + ] + +class AlertInfoInline(admin.StackedInline): + model = AlertInfo + extra = 0 + class AlertAdmin(admin.ModelAdmin): - list_display = ["id", "source", "urgency", "severity", "certainty", "sent", "effective", "expires"] - list_filter = ["source", "urgency", "severity", "certainty", "sent", "effective", "expires"] + list_display = ["id", "source_feed", "sent"] + list_filter = ["source_feed", "country"] + fieldsets = [ + ("Administration", {"fields": ["country", "source_feed"]}), + ("Alert Header" , {"fields": ["identifier", "sender", "sent", "status", "msg_type", "source", "scope", "restriction", "addresses", "code", "note", "references", "incidents"]}), + ] + inlines = [AlertInfoInline] class CountryAdmin(admin.ModelAdmin): list_display = ["name", "iso3", "region", "continent"] @@ -22,6 +39,7 @@ class SourceAdmin(admin.ModelAdmin): admin.site.register(Alert, AlertAdmin) +admin.site.register(AlertInfo, AlertInfoAdmin) admin.site.register(Continent) admin.site.register(Region) admin.site.register(Country, CountryAdmin) diff --git a/cap_feed/alert_processing.py b/cap_feed/alert_processing.py deleted file mode 100644 index 4a40066..0000000 --- a/cap_feed/alert_processing.py +++ /dev/null @@ -1,224 +0,0 @@ -import os -module_dir = os.path.dirname(__file__) # get current directory -import json -import requests - -import xml.etree.ElementTree as ET -import pytz -from .models import Alert, Continent, Region, Country, Source -from datetime import datetime -from django.utils import timezone - -# inject source configurations if not already present -def inject_sources(): - source_data = [ - ("https://feeds.meteoalarm.org/feeds/meteoalarm-legacy-atom-france", "FRA", "meteoalarm", {'atom': 'http://www.w3.org/2005/Atom', 'cap': 'urn:oasis:names:tc:emergency:cap:1.2'}), - ("https://feeds.meteoalarm.org/feeds/meteoalarm-legacy-atom-belgium", "BEL", "meteoalarm", {'atom': 'http://www.w3.org/2005/Atom', 'cap': 'urn:oasis:names:tc:emergency:cap:1.2'}), - ("https://feeds.meteoalarm.org/feeds/meteoalarm-legacy-atom-austria", "AUT", "meteoalarm", {'atom': 'http://www.w3.org/2005/Atom', 'cap': 'urn:oasis:names:tc:emergency:cap:1.2'}), - ("https://feeds.meteoalarm.org/feeds/meteoalarm-legacy-atom-slovakia", "SVK", "meteoalarm", {'atom': 'http://www.w3.org/2005/Atom', 'cap': 'urn:oasis:names:tc:emergency:cap:1.2'}), - ("https://feeds.meteoalarm.org/feeds/meteoalarm-legacy-atom-slovenia", "SVN", "meteoalarm", {'atom': 'http://www.w3.org/2005/Atom', 'cap': 'urn:oasis:names:tc:emergency:cap:1.2'}), - ("https://feeds.meteoalarm.org/feeds/meteoalarm-legacy-atom-bosnia-herzegovina", "BIH", "meteoalarm", {'atom': 'http://www.w3.org/2005/Atom', 'cap': 'urn:oasis:names:tc:emergency:cap:1.2'}), - ("https://feeds.meteoalarm.org/feeds/meteoalarm-legacy-atom-bulgaria", "BGR", "meteoalarm", {'atom': 'http://www.w3.org/2005/Atom', 'cap': 'urn:oasis:names:tc:emergency:cap:1.2'}), - ("https://feeds.meteoalarm.org/feeds/meteoalarm-legacy-atom-croatia", "HRV", "meteoalarm", {'atom': 'http://www.w3.org/2005/Atom', 'cap': 'urn:oasis:names:tc:emergency:cap:1.2'}), - ("https://feeds.meteoalarm.org/feeds/meteoalarm-legacy-atom-cyprus", "CYP", "meteoalarm", {'atom': 'http://www.w3.org/2005/Atom', 'cap': 'urn:oasis:names:tc:emergency:cap:1.2'}), - ("https://feeds.meteoalarm.org/feeds/meteoalarm-legacy-atom-czechia", "CZE", "meteoalarm", {'atom': 'http://www.w3.org/2005/Atom', 'cap': 'urn:oasis:names:tc:emergency:cap:1.2'}), - ("https://feeds.meteoalarm.org/feeds/meteoalarm-legacy-atom-denmark", "DNK", "meteoalarm", {'atom': 'http://www.w3.org/2005/Atom', 'cap': 'urn:oasis:names:tc:emergency:cap:1.2'}), - ("https://feeds.meteoalarm.org/feeds/meteoalarm-legacy-atom-estonia", "EST", "meteoalarm", {'atom': 'http://www.w3.org/2005/Atom', 'cap': 'urn:oasis:names:tc:emergency:cap:1.2'}), - ("https://feeds.meteoalarm.org/feeds/meteoalarm-legacy-atom-finland", "FIN", "meteoalarm", {'atom': 'http://www.w3.org/2005/Atom', 'cap': 'urn:oasis:names:tc:emergency:cap:1.2'}), - ("https://feeds.meteoalarm.org/feeds/meteoalarm-legacy-atom-greece", "GRC", "meteoalarm", {'atom': 'http://www.w3.org/2005/Atom', 'cap': 'urn:oasis:names:tc:emergency:cap:1.2'}), - ("https://feeds.meteoalarm.org/feeds/meteoalarm-legacy-atom-hungary", "HUN", "meteoalarm", {'atom': 'http://www.w3.org/2005/Atom', 'cap': 'urn:oasis:names:tc:emergency:cap:1.2'}), - ("https://alert.metservice.gov.jm/capfeed.php", "JAM", "capfeedphp", {'atom': 'http://www.w3.org/2005/Atom', 'cap': 'urn:oasis:names:tc:emergency:cap:1.2'}), - ] - - if Source.objects.count() == 0: - for source_entry in source_data: - source = Source() - source.url = source_entry[0] - source.polling_interval = 60 - source.country = Country.objects.get(iso3 = source_entry[1]) - source.format = source_entry[2] - source.atom = source_entry[3]['atom'] - source.cap = source_entry[3]['cap'] - source.save() - -# inject region and country data if not already present -def inject_geographical_data(): - if Continent.objects.count() == 0: - inject_continents() - if Region.objects.count() == 0: - inject_regions() - if Country.objects.count() == 0: - inject_countries() - -# inject continent data -def inject_continents(): - file_path = os.path.join(module_dir, 'geographical/continents.json') - with open(file_path) as file: - continent_data = json.load(file) - for continent_entry in continent_data: - continent = Continent() - continent.name = continent_entry["name"] - continent.save() - -# inject region data -def inject_regions(): - file_path = os.path.join(module_dir, 'geographical/ifrc-regions.json') - with open(file_path) as file: - region_data = json.load(file) - for region_entry in region_data: - region = Region() - region.name = region_entry["region_name"] - region.centroid = region_entry["centroid"] - coordinates = region_entry["bbox"]["coordinates"][0] - for coordinate in coordinates: - region.polygon += str(coordinate[0]) + "," + str(coordinate[1]) + " " - region.save() - -# inject country data -def inject_countries(): - region_names = {} - file_path = os.path.join(module_dir, 'geographical/ifrc-regions.json') - with open(file_path) as file: - region_data = json.load(file) - for region_entry in region_data: - name = region_entry["region_name"] - region_id = region_entry["id"] - region_names[region_id] = name - - ifrc_countries = {} - file_path = os.path.join(module_dir, 'geographical/ifrc-countries-and-territories.json') - with open(file_path) as file: - country_data = json.load(file) - for feature in country_data: - name = feature["name"] - region_id = feature["region"] - iso3 = feature["iso3"] - if ("Region" in name) or ("Cluster" in name) or (region_id is None) or (iso3 is None): - continue - ifrc_countries[iso3] = region_names[region_id] - - processed_iso3 = set() - file_path = os.path.join(module_dir, 'geographical/opendatasoft-countries-and-territories.geojson') - with open(file_path) as file: - country_data = json.load(file) - for feature in country_data['features']: - country = Country() - country.name = feature['properties']['name'] - country.iso3 = feature['properties']['iso3'] - status = feature['properties']['status'] - if status == 'Occupied Territory (under review)' or status == 'PT Territory': - continue - if (country.iso3 in ifrc_countries): - country.region = Region.objects.filter(name = ifrc_countries[country.iso3]).first() - country.continent = Continent.objects.filter(name = feature['properties']['continent']).first() - coordinates = feature['geometry']['coordinates'] - if len(coordinates) == 1: - country.polygon = coordinates - else: - country.multipolygon = coordinates - - latitude = feature['properties']['geo_point_2d']['lat'] - longitude = feature['properties']['geo_point_2d']['lon'] - country.centroid = f'[{longitude}, {latitude}]' - - #"properties":{"geo_point_2d":{"lon":31.49752845618843,"lat":-26.562642190929807}, - - country.save() - processed_iso3.add(country.iso3) - -# converts CAP1.2 iso format datetime string to datetime object in UTC timezone -def convert_datetime(original_datetime): - return datetime.fromisoformat(original_datetime).astimezone(pytz.timezone('UTC')) - -# gets alerts from sources and processes them different for each source format -def poll_new_alerts(sources): - # list of sources and configurations - for source in sources: - url = source["url"] - iso3 = Country.objects.get(name = source["country"]).iso3 - format = source["format"] - ns = {"atom":source["atom"], "cap": source["cap"]} - match format: - case "meteoalarm": - get_alert_meteoalarm(url, iso3, ns) - case "capfeedphp": - get_alert_capfeedphp(url, iso3, ns) - -# processing for meteoalarm format, example: https://feeds.meteoalarm.org/feeds/meteoalarm-legacy-atom-france -def get_alert_meteoalarm(url, iso3, ns): - response = requests.get(url) - root = ET.fromstring(response.content) - for entry in root.findall('atom:entry', ns): - try: - alert = Alert() - alert.source = Source.objects.get(url=url) - - alert.id = entry.find('atom:id', ns).text - alert.identifier = entry.find('cap:identifier', ns).text - - #sender needs to be fixed - alert.sender = url - alert.sent = convert_datetime(entry.find('cap:sent', ns).text) - alert.status = entry.find('cap:status', ns).text - alert.msg_type = entry.find('cap:message_type', ns).text - alert.scope = entry.find('cap:scope', ns).text - alert.urgency = entry.find('cap:urgency', ns).text - alert.severity = entry.find('cap:severity', ns).text - alert.certainty = entry.find('cap:certainty', ns).text - alert.effective = convert_datetime(entry.find('cap:effective', ns).text) - alert.expires = convert_datetime(entry.find('cap:expires', ns).text) - if alert.expires < timezone.now(): - continue - - alert.area_desc = entry.find('cap:areaDesc', ns).text - alert.event = entry.find('cap:event', ns).text - geocode = entry.find('cap:geocode', ns) - if geocode is not None: - alert.geocode_name = geocode.find('atom:valueName', ns).text - alert.geocode_value = geocode.find('atom:value', ns).text - alert.country = Country.objects.get(iso3=iso3) - alert.save() - except Exception as e: - print("get_alert_meteoalarm", e) - -# processing for capfeedphp format, example: https://alert.metservice.gov.jm/capfeed.php -def get_alert_capfeedphp(url, iso3, ns): - response = requests.get(url) - root = ET.fromstring(response.content) - for entry in root.findall('atom:entry', ns): - try: - alert = Alert() - alert.source = Source.objects.get(url=url) - - alert.id = entry.find('atom:id', ns).text - entry_content = entry.find('atom:content', ns) - entry_content_alert = entry_content.find('cap:alert', ns) - alert.identifier = entry_content_alert.find('cap:identifier', ns).text - alert.sender = entry_content_alert.find('cap:sender', ns).text - alert.sent = convert_datetime(entry_content_alert.find('cap:sent', ns).text) - alert.status = entry_content_alert.find('cap:status', ns).text - alert.msg_type = entry_content_alert.find('cap:msgType', ns).text - alert.scope = entry_content_alert.find('cap:scope', ns).text - - entry_content_alert_info = entry_content_alert.find('cap:info', ns) - alert.event = entry_content_alert_info.find('cap:event', ns).text - alert.urgency = entry_content_alert_info.find('cap:urgency', ns).text - alert.severity = entry_content_alert_info.find('cap:severity', ns).text - alert.certainty = entry_content_alert_info.find('cap:certainty', ns).text - alert.effective = convert_datetime(entry_content_alert_info.find('cap:effective', ns).text) - alert.expires = convert_datetime(entry_content_alert_info.find('cap:expires', ns).text) - if alert.expires < timezone.now(): - continue - alert.senderName = entry_content_alert_info.find('cap:senderName', ns).text - alert.description = entry_content_alert_info.find('cap:description', ns).text - - entry_content_alert_info_area = entry_content_alert_info.find('cap:area', ns) - alert.area_desc = entry_content_alert_info_area.find('cap:areaDesc', ns).text - #alert.polygon = entry_content_alert_info_area.find('cap:polygon', ns).text - alert.country = Country.objects.get(iso3=iso3) - alert.save() - except Exception as e: - print("get_alert_capfeedphp", e) - -def remove_expired_alerts(): - Alert.objects.filter(expires__lt=timezone.now()).delete() \ No newline at end of file diff --git a/cap_feed/alert_processor.py b/cap_feed/alert_processor.py new file mode 100644 index 0000000..373965c --- /dev/null +++ b/cap_feed/alert_processor.py @@ -0,0 +1,19 @@ +from .models import Alert, AlertInfo, Country +from django.utils import timezone +import cap_feed.formats.format_handler as fh + + + +# gets alerts from sources and processes them different for each source format +def poll_new_alerts(sources): + # list of sources and configurations + for source in sources: + format = source['format'] + url = source['url'] + country = Country.objects.get(iso3=source['country']) + ns = {"atom": source['atom'], "cap": source['cap']} + fh.get_alerts(format, url, country, ns) + +def remove_expired_alerts(): + AlertInfo.objects.filter(expires__lt=timezone.now()).delete() + Alert.objects.filter(alertinfo__isnull=True).delete() \ No newline at end of file diff --git a/cap_feed/data_injector.py b/cap_feed/data_injector.py new file mode 100644 index 0000000..03e6f73 --- /dev/null +++ b/cap_feed/data_injector.py @@ -0,0 +1,121 @@ +import os +import json +module_dir = os.path.dirname(__file__) # get current directory +from .models import Alert, Continent, Region, Country, Source +from cap_feed.formats import format_handler as fh + + + +# inject region and country data if not already present +def inject_geographical_data(): + if Continent.objects.count() == 0: + inject_continents() + if Region.objects.count() == 0: + inject_regions() + if Country.objects.count() == 0: + inject_countries() + +# inject continent data +def inject_continents(): + file_path = os.path.join(module_dir, 'geographical/continents.json') + with open(file_path) as file: + continent_data = json.load(file) + for continent_entry in continent_data: + continent = Continent() + continent.name = continent_entry["name"] + continent.save() + +# inject continent data +def inject_continents(): + file_path = os.path.join(module_dir, 'geographical/continents.json') + with open(file_path) as file: + continent_data = json.load(file) + for continent_entry in continent_data: + continent = Continent() + continent.id = continent_entry["id"] + continent.name = continent_entry["name"] + continent.save() + +# inject region data +def inject_regions(): + file_path = os.path.join(module_dir, 'geographical/ifrc-regions.json') + with open(file_path) as file: + region_data = json.load(file) + for region_entry in region_data: + region = Region() + region.name = region_entry["region_name"] + region.centroid = region_entry["centroid"] + coordinates = region_entry["bbox"]["coordinates"][0] + for coordinate in coordinates: + region.polygon += str(coordinate[0]) + "," + str(coordinate[1]) + " " + region.save() + +# inject country data +def inject_countries(): + region_names = {} + file_path = os.path.join(module_dir, 'geographical/ifrc-regions.json') + with open(file_path) as file: + region_data = json.load(file) + for region_entry in region_data: + name = region_entry["region_name"] + region_id = region_entry["id"] + region_names[region_id] = name + + ifrc_countries = {} + file_path = os.path.join(module_dir, 'geographical/ifrc-countries-and-territories.json') + with open(file_path) as file: + country_data = json.load(file) + for feature in country_data: + name = feature["name"] + region_id = feature["region"] + iso3 = feature["iso3"] + if ("Region" in name) or ("Cluster" in name) or (region_id is None) or (iso3 is None): + continue + ifrc_countries[iso3] = region_names[region_id] + + processed_iso3 = set() + file_path = os.path.join(module_dir, 'geographical/opendatasoft-countries-and-territories.geojson') + with open(file_path) as file: + country_data = json.load(file) + for feature in country_data['features']: + country = Country() + country.name = feature['properties']['name'] + country.iso3 = feature['properties']['iso3'] + status = feature['properties']['status'] + if status == 'Occupied Territory (under review)' or status == 'PT Territory': + continue + if (country.iso3 in ifrc_countries): + country.region = Region.objects.filter(name = ifrc_countries[country.iso3]).first() + country.continent = Continent.objects.filter(name = feature['properties']['continent']).first() + coordinates = feature['geometry']['coordinates'] + if len(coordinates) == 1: + country.polygon = coordinates + else: + country.multipolygon = coordinates + + latitude = feature['properties']['geo_point_2d']['lat'] + longitude = feature['properties']['geo_point_2d']['lon'] + country.centroid = f'[{longitude}, {latitude}]' + + #"properties":{"geo_point_2d":{"lon":31.49752845618843,"lat":-26.562642190929807}, + + country.save() + processed_iso3.add(country.iso3) + +# inject source configurations if not already present +def inject_sources(): + # this could be converted to a fixture + source_data = [ + ("https://feeds.meteoalarm.org/feeds/meteoalarm-legacy-atom-france", "FRA", "meteoalarm"), + ("https://cap-sources.s3.amazonaws.com/mg-meteo-en/rss.xml", "MDG", "aws"), + ("https://cap-sources.s3.amazonaws.com/cm-meteo-en/rss.xml", "CMR", "aws"), + ("https://api.weather.gov/alerts/active", "USA", "nws_us"), + ] + + for source_entry in source_data: + source = Source() + source.url = source_entry[0] + source.polling_interval = 60 + source.country = Country.objects.get(iso3 = source_entry[1]) + source.format = source_entry[2] + source.save() \ No newline at end of file diff --git a/cap_feed/formats/__init__.py b/cap_feed/formats/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/cap_feed/formats/aws.py b/cap_feed/formats/aws.py new file mode 100644 index 0000000..4226c8e --- /dev/null +++ b/cap_feed/formats/aws.py @@ -0,0 +1,64 @@ +import requests +import xml.etree.ElementTree as ET +from django.utils import timezone +from cap_feed.models import Alert, AlertInfo, Source +from cap_feed.formats.utils import convert_datetime + + + +# processing for aws format, example: https://cap-sources.s3.amazonaws.com/mg-meteo-en/rss.xml +def get_alerts_aws(url, country, ns): + # navigate list of alerts + response = requests.get(url) + root = ET.fromstring(response.content) + for alert_entry in root.find('channel').findall('item'): + try: + # skip if alert already exists + id = alert_entry.find('link').text + if Alert.objects.filter(id=id).exists(): + continue + + # register intial alert details + alert = Alert() + alert.source_feed = Source.objects.get(url=url) + alert.country = country + alert.id = id + + # navigate alert + alert_response = requests.get(alert.id) + alert_root = ET.fromstring(alert_response.content) + alert.identifier = alert_root.find('cap:identifier', ns).text + alert.sender = alert_root.find('cap:sender', ns).text + alert.sent = convert_datetime(alert_root.find('cap:sent', ns).text) + alert.status = alert_root.find('cap:status', ns).text + alert.msg_type = alert_root.find('cap:msgType', ns).text + alert.scope = alert_root.find('cap:scope', ns).text + + # navigate alert info + for alert_info_entry in alert_root.findall('cap:info', ns): + alert_info = AlertInfo() + alert_info.alert = alert + alert_info.language = alert_info_entry.find('cap:language', ns).text + alert_info.category = alert_info_entry.find('cap:category', ns).text + alert_info.event = alert_info_entry.find('cap:event', ns).text + alert_info.response_type = alert_info_entry.find('cap:responseType', ns).text + alert_info.urgency = alert_info_entry.find('cap:urgency', ns).text + alert_info.severity = alert_info_entry.find('cap:severity', ns).text + alert_info.certainty = alert_info_entry.find('cap:certainty', ns).text + alert_info.effective = alert.sent if (x := alert_info_entry.find('cap:effective', ns)) is None else x.text + alert_info.onset = convert_datetime(alert_info_entry.find('cap:onset', ns).text) + alert_info.expires = convert_datetime(alert_info_entry.find('cap:expires', ns).text) + if alert_info.expires < timezone.now(): + continue + alert_info.sender_name = alert_info_entry.find('cap:senderName', ns).text + alert_info.headline = alert_info_entry.find('cap:headline', ns).text + alert_info.description = alert_info_entry.find('cap:description', ns).text + alert_info.instruction = alert_info_entry.find('cap:instruction', ns).text + alert_info.web = alert_info_entry.find('cap:web', ns).text + alert_info.contact = alert_info_entry.find('cap:contact', ns).text + alert.save() + alert_info.save() + + except Exception as e: + print("get_alerts_aws", e) + print("id:", id) \ No newline at end of file diff --git a/cap_feed/formats/format_handler.py b/cap_feed/formats/format_handler.py new file mode 100644 index 0000000..d53ef32 --- /dev/null +++ b/cap_feed/formats/format_handler.py @@ -0,0 +1,16 @@ +from .meteoalarm import get_alerts_meteoalarm +from .aws import get_alerts_aws +from .nws_us import get_alerts_nws_us + + + +def get_alerts(format, url, country, ns): + match format: + case "meteoalarm": + get_alerts_meteoalarm(url, country, ns) + case "aws": + get_alerts_aws(url, country, ns) + case "nws_us": + get_alerts_nws_us(url, country, ns) + case _: + print("Format not supported") \ No newline at end of file diff --git a/cap_feed/formats/meteoalarm.py b/cap_feed/formats/meteoalarm.py new file mode 100644 index 0000000..7c6f5e6 --- /dev/null +++ b/cap_feed/formats/meteoalarm.py @@ -0,0 +1,63 @@ +import requests +import xml.etree.ElementTree as ET +from django.utils import timezone +from cap_feed.models import Alert, AlertInfo, Source +from cap_feed.formats.utils import convert_datetime + + + +# processing for meteoalarm format, example: https://feeds.meteoalarm.org/feeds/meteoalarm-legacy-atom-france +def get_alerts_meteoalarm(url, country, ns): + # navigate list of alerts + response = requests.get(url) + root = ET.fromstring(response.content) + for alert_entry in root.findall('atom:entry', ns): + try: + # skip if alert is expired or already exists + expires = convert_datetime(alert_entry.find('cap:expires', ns).text) + id = alert_entry.find('atom:id', ns).text + if expires < timezone.now() or Alert.objects.filter(id=id).exists(): + continue + + # register intial alert details + alert = Alert() + alert.source_feed = Source.objects.get(url=url) + alert.country = country + alert.id = id + + # navigate alert + alert_response = requests.get(alert.id) + alert_root = ET.fromstring(alert_response.content) + alert.identifier = alert_root.find('cap:identifier', ns).text + alert.sender = alert_root.find('cap:sender', ns).text + alert.sent = convert_datetime(alert_root.find('cap:sent', ns).text) + alert.status = alert_root.find('cap:status', ns).text + alert.msg_type = alert_root.find('cap:msgType', ns).text + alert.scope = alert_root.find('cap:scope', ns).text + alert.save() + + # navigate alert info + for alert_info_entry in alert_root.findall('cap:info', ns): + alert_info = AlertInfo() + alert_info.alert = alert + alert_info.language = alert_info_entry.find('cap:language', ns).text + alert_info.category = alert_info_entry.find('cap:category', ns).text + alert_info.event = alert_info_entry.find('cap:event', ns).text + alert_info.response_type = alert_info_entry.find('cap:responseType', ns).text + alert_info.urgency = alert_info_entry.find('cap:urgency', ns).text + alert_info.severity = alert_info_entry.find('cap:severity', ns).text + alert_info.certainty = alert_info_entry.find('cap:certainty', ns).text + alert_info.effective = alert.sent if (x := alert_info_entry.find('cap:effective', ns)) is None else x.text + alert_info.onset = convert_datetime(alert_info_entry.find('cap:onset', ns).text) + alert_info.expires = convert_datetime(alert_info_entry.find('cap:expires', ns).text) + alert_info.sender_name = alert_info_entry.find('cap:senderName', ns).text + alert_info.headline = alert_info_entry.find('cap:headline', ns).text + alert_info.description = alert_info_entry.find('cap:description', ns).text + alert_info.instruction = alert_info_entry.find('cap:instruction', ns).text + alert_info.web = alert_info_entry.find('cap:web', ns).text + alert_info.contact = alert_info_entry.find('cap:contact', ns).text + alert_info.save() + + except Exception as e: + print("get_alerts_meteoalarm", e) + print("id:", id) \ No newline at end of file diff --git a/cap_feed/formats/nws_us.py b/cap_feed/formats/nws_us.py new file mode 100644 index 0000000..a1c8745 --- /dev/null +++ b/cap_feed/formats/nws_us.py @@ -0,0 +1,67 @@ +import requests +import xml.etree.ElementTree as ET +from django.utils import timezone +from cap_feed.models import Alert, AlertInfo, Source +from cap_feed.formats.utils import convert_datetime + + + +# processing for nws_us format, example: https://api.weather.gov/alerts/active +def get_alerts_nws_us(url, country, ns): + # navigate list of alerts + response = requests.get(url, headers={'Accept': 'application/atom+xml'}) + root = ET.fromstring(response.content) + + for alert_entry in root.findall('atom:entry', ns): + try: + # skip if alert is expired or already exists + expires = convert_datetime(alert_entry.find('cap:expires', ns).text) + id = alert_entry.find('atom:id', ns).text + if expires < timezone.now() or Alert.objects.filter(id=id).exists(): + continue + + # register intial alert details + alert = Alert() + alert.source_feed = Source.objects.get(url=url) + alert.country = country + alert.id = id + + # navigate alert + cap_link = alert_entry.find('atom:link', ns).attrib['href'] + alert_response = requests.get(cap_link) + alert_root = ET.fromstring(alert_response.content) + alert.identifier = alert_root.find('cap:identifier', ns).text + alert.sender = alert_root.find('cap:sender', ns).text + alert.sent = convert_datetime(alert_root.find('cap:sent', ns).text) + alert.status = alert_root.find('cap:status', ns).text + alert.msg_type = alert_root.find('cap:msgType', ns).text + alert.scope = alert_root.find('cap:scope', ns).text + alert.code = alert_root.find('cap:code', ns).text + if (x := root.find('cap:references', ns)) is not None: alert.references = x.text + alert.save() + + # navigate alert info + for alert_info_entry in alert_root.findall('cap:info', ns): + alert_info = AlertInfo() + alert_info.alert = alert + alert_info.language = alert_info_entry.find('cap:language', ns).text + alert_info.category = alert_info_entry.find('cap:category', ns).text + alert_info.event = alert_info_entry.find('cap:event', ns).text + alert_info.response_type = alert_info_entry.find('cap:responseType', ns).text + alert_info.urgency = alert_info_entry.find('cap:urgency', ns).text + alert_info.severity = alert_info_entry.find('cap:severity', ns).text + alert_info.certainty = alert_info_entry.find('cap:certainty', ns).text + #alert_info.event_code + alert_info.effective = alert.sent if (x := alert_info_entry.find('cap:effective', ns)) is None else x.text + if (x := alert_info_entry.find('cap:onset', ns)) is not None: alert_info.onset = convert_datetime(x.text) + alert_info.expires = convert_datetime(alert_info_entry.find('cap:expires', ns).text) + if (x := alert_info_entry.find('cap:senderName', ns)) is not None: alert_info.sender_name = x.text + if (x := alert_info_entry.find('cap:headline', ns)) is not None: alert_info.headline = x.text + if (x := alert_info_entry.find('cap:description', ns)) is not None: alert_info.description = x.text + if (x := alert_info_entry.find('cap:instruction', ns)) is not None: alert_info.instruction = x.text + alert_info.web = alert_info_entry.find('cap:web', ns).text + alert_info.save() + + except Exception as e: + print("get_alerts_nws_us:", e) + print("id:", id) \ No newline at end of file diff --git a/cap_feed/formats/utils.py b/cap_feed/formats/utils.py new file mode 100644 index 0000000..f970287 --- /dev/null +++ b/cap_feed/formats/utils.py @@ -0,0 +1,8 @@ +from datetime import datetime +import pytz + + + +# converts CAP1.2 iso format datetime string to datetime object in UTC timezone +def convert_datetime(original_datetime): + return datetime.fromisoformat(original_datetime).astimezone(pytz.timezone('UTC')) \ No newline at end of file diff --git a/cap_feed/migrations/0015_alter_alert_identifier_alter_alert_scope_and_more.py b/cap_feed/migrations/0015_alter_alert_identifier_alter_alert_scope_and_more.py new file mode 100644 index 0000000..0d93cb0 --- /dev/null +++ b/cap_feed/migrations/0015_alter_alert_identifier_alter_alert_scope_and_more.py @@ -0,0 +1,33 @@ +# Generated by Django 4.2.3 on 2023-07-07 19:56 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('cap_feed', '0014_alert_country_source_country'), + ] + + operations = [ + migrations.AlterField( + model_name='alert', + name='identifier', + field=models.CharField(default='', max_length=255), + ), + migrations.AlterField( + model_name='alert', + name='scope', + field=models.CharField(default='', max_length=255), + ), + migrations.AlterField( + model_name='alert', + name='sent', + field=models.DateTimeField(default=(1970, 1, 1, 0, 0, 0, 3, 1, 0)), + ), + migrations.AlterField( + model_name='source', + name='format', + field=models.CharField(choices=[('meteoalarm', 'meteoalarm'), ('capfeedphp', 'capfeedphp'), ('capusphp', 'capusphp')]), + ), + ] diff --git a/cap_feed/migrations/0016_remove_alert_area_desc_remove_alert_certainty_and_more.py b/cap_feed/migrations/0016_remove_alert_area_desc_remove_alert_certainty_and_more.py new file mode 100644 index 0000000..4aa583c --- /dev/null +++ b/cap_feed/migrations/0016_remove_alert_area_desc_remove_alert_certainty_and_more.py @@ -0,0 +1,152 @@ +# Generated by Django 4.2.3 on 2023-07-08 20:12 + +import datetime +from django.db import migrations, models +import django.db.models.deletion +import django.utils.timezone + + +class Migration(migrations.Migration): + + dependencies = [ + ('cap_feed', '0015_alter_alert_identifier_alter_alert_scope_and_more'), + ] + + operations = [ + migrations.RemoveField( + model_name='alert', + name='area_desc', + ), + migrations.RemoveField( + model_name='alert', + name='certainty', + ), + migrations.RemoveField( + model_name='alert', + name='description', + ), + migrations.RemoveField( + model_name='alert', + name='effective', + ), + migrations.RemoveField( + model_name='alert', + name='event', + ), + migrations.RemoveField( + model_name='alert', + name='expires', + ), + migrations.RemoveField( + model_name='alert', + name='geocode_name', + ), + migrations.RemoveField( + model_name='alert', + name='geocode_value', + ), + migrations.RemoveField( + model_name='alert', + name='senderName', + ), + migrations.RemoveField( + model_name='alert', + name='severity', + ), + migrations.RemoveField( + model_name='alert', + name='urgency', + ), + migrations.AddField( + model_name='alert', + name='addresses', + field=models.TextField(null=True), + ), + migrations.AddField( + model_name='alert', + name='code', + field=models.CharField(max_length=255, null=True), + ), + migrations.AddField( + model_name='alert', + name='incidents', + field=models.TextField(null=True), + ), + migrations.AddField( + model_name='alert', + name='note', + field=models.TextField(null=True), + ), + migrations.AddField( + model_name='alert', + name='references', + field=models.TextField(null=True), + ), + migrations.AddField( + model_name='alert', + name='restriction', + field=models.CharField(max_length=255, null=True), + ), + migrations.AddField( + model_name='alert', + name='source_feed', + field=models.ForeignKey(default=1, on_delete=django.db.models.deletion.CASCADE, to='cap_feed.source'), + preserve_default=False, + ), + migrations.AlterField( + model_name='alert', + name='identifier', + field=models.CharField(max_length=255), + ), + migrations.AlterField( + model_name='alert', + name='msg_type', + field=models.CharField(choices=[('Alert', 'Alert'), ('Update', 'Update'), ('Cancel', 'Cancel'), ('Ack', 'Ack'), ('Error', 'Error')]), + ), + migrations.AlterField( + model_name='alert', + name='scope', + field=models.CharField(choices=[('Public', 'Public'), ('Restricted', 'Restricted'), ('Private', 'Private')]), + ), + migrations.AlterField( + model_name='alert', + name='sent', + field=models.DateTimeField(), + ), + migrations.AlterField( + model_name='alert', + name='source', + field=models.CharField(max_length=255, null=True), + ), + migrations.AlterField( + model_name='alert', + name='status', + field=models.CharField(choices=[('Actual', 'Actual'), ('Exercise', 'Exercise'), ('System', 'System'), ('Test', 'Test'), ('Draft', 'Draft')]), + ), + migrations.CreateModel( + name='AlertInfo', + fields=[ + ('id', models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('language', models.CharField(default='en-US', max_length=255)), + ('category', models.CharField(choices=[('Geo', 'Geo'), ('capfeedphp', 'Met'), ('Safety', 'Safety'), ('Security', 'Security'), ('Rescue', 'Rescue'), ('Fire', 'Fire'), ('Health', 'Health'), ('Env', 'Env'), ('Transport', 'Transport'), ('Infra', 'Infra'), ('CBRNE', 'CBRNE'), ('Other', 'Other')])), + ('event', models.CharField(max_length=255)), + ('response_type', models.CharField(choices=[('Shelter', 'Shelter'), ('Evacuate', 'Evacuate'), ('Prepare', 'Prepare'), ('Avoid', 'Avoid'), ('Monitor', 'Monitor'), ('Assess', 'Assess'), ('AllClear', 'AllClear'), ('None', 'None')], null=True)), + ('urgency', models.CharField(choices=[('Immediate', 'Immediate'), ('Expected', 'Expected'), ('Future', 'Future'), ('Past', 'Past'), ('Unknown', 'Unknown')])), + ('severity', models.CharField(choices=[('Extreme', 'Extreme'), ('Severe', 'Severe'), ('Moderate', 'Moderate'), ('Minor', 'Minor'), ('Unknown', 'Unknown')])), + ('certainty', models.CharField(choices=[('Observed', 'Observed'), ('Likely', 'Likely'), ('Possible', 'Possible'), ('Unlikely', 'Unlikely'), ('Unknown', 'Unknown')])), + ('audience', models.CharField(null=True)), + ('event_code', models.CharField(max_length=255, null=True)), + ('effective', models.DateTimeField(default=django.utils.timezone.now)), + ('onset', models.DateTimeField(null=True)), + ('expires', models.DateTimeField(default=datetime.datetime(2023, 7, 9, 20, 12, 13, 728041, tzinfo=datetime.timezone.utc))), + ('sender_name', models.CharField(max_length=255, null=True)), + ('headline', models.CharField(max_length=255, null=True)), + ('description', models.TextField(null=True)), + ('instruction', models.TextField(null=True)), + ('web', models.URLField(null=True)), + ('contact', models.CharField(max_length=255, null=True)), + ('parameter', models.CharField(max_length=255, null=True)), + ('alert', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='cap_feed.alert')), + ], + ), + ] diff --git a/cap_feed/models.py b/cap_feed/models.py index e758a42..d9a30c8 100644 --- a/cap_feed/models.py +++ b/cap_feed/models.py @@ -1,6 +1,8 @@ from django.core.validators import MaxValueValidator, MinValueValidator from django.db import models import json +import time +from datetime import timedelta from django.utils import timezone @@ -43,15 +45,16 @@ class Source(models.Model): FORMAT_CHOICES = [ ('meteoalarm', 'meteoalarm'), - ('capfeedphp', 'capfeedphp') + ('aws', 'aws'), + ('nws_us', 'nws_us') ] url = models.CharField(primary_key=True, max_length=255) country = models.ForeignKey(Country, on_delete=models.CASCADE) format = models.CharField(choices=FORMAT_CHOICES) polling_interval = models.IntegerField(choices=INTERVAL_CHOICES) - atom = models.CharField(max_length=255) - cap = models.CharField(max_length=255) + atom = models.CharField(editable=False, default='http://www.w3.org/2005/Atom') + cap = models.CharField(editable=False, default='urn:oasis:names:tc:emergency:cap:1.2') __previous_polling_interval = None __previous_url = None @@ -72,16 +75,16 @@ def save(self, force_insert=False, force_update=False, *args, **kwargs): update_source(self, self.__previous_url, self.__previous_polling_interval) super(Source, self).save(force_insert, force_update, *args, **kwargs) - #This function is to be used for serialisation + # For serialization def to_dict(self): - dictionary = dict() - dictionary['url'] = self.url - dictionary['polling_interval'] = self.polling_interval - dictionary['country'] = self.country.name - dictionary['format'] = self.format - dictionary['atom'] = self.atom - dictionary['cap'] = self.cap - return dictionary + source_dict = dict() + source_dict['url'] = self.url + source_dict['country'] = self.country.iso3 + source_dict['format'] = self.format + source_dict['polling_interval'] = self.polling_interval + source_dict['atom'] = self.atom + source_dict['cap'] = self.cap + return source_dict class SourceEncoder(json.JSONEncoder): def default(self, obj): @@ -91,81 +94,69 @@ def default(self, obj): return super().default(obj) class Alert(models.Model): - id = models.CharField(max_length=255, primary_key=True) - identifier = models.CharField(max_length=255) - sender = models.CharField(max_length=255) - senderName = models.CharField(max_length=255, default='') - source = models.ForeignKey(Source, on_delete=models.CASCADE) - sent = models.DateTimeField() - status = models.CharField(max_length=255) - msg_type = models.CharField(max_length=255) - scope = models.CharField(max_length=255) - urgency = models.CharField(max_length=255) - severity = models.CharField(max_length=255) - certainty = models.CharField(max_length=255) - effective = models.DateTimeField() - expires = models.DateTimeField() + STATUS_CHOICES = [ + ('Actual', 'Actual'), + ('Exercise', 'Exercise'), + ('System', 'System'), + ('Test', 'Test'), + ('Draft', 'Draft') + ] - description = models.TextField(blank=True, default='') + MSG_TYPE_CHOICES = [ + ('Alert', 'Alert'), + ('Update', 'Update'), + ('Cancel', 'Cancel'), + ('Ack', 'Ack'), + ('Error', 'Error') + ] + + SCOPE_CHOICES = [ + ('Public', 'Public'), + ('Restricted', 'Restricted'), + ('Private', 'Private') + ] - area_desc = models.CharField(max_length=255) - event = models.CharField(max_length=255) - geocode_name = models.CharField(max_length=255, blank=True, default='') - geocode_value = models.CharField(max_length=255, blank=True, default='') country = models.ForeignKey(Country, on_delete=models.CASCADE) + source_feed = models.ForeignKey(Source, on_delete=models.CASCADE) + id = models.CharField(primary_key=True, max_length=255) + + identifier = models.CharField(max_length=255) + sender = models.CharField(max_length=255) + sent = models.DateTimeField() + status = models.CharField(choices = STATUS_CHOICES) + msg_type = models.CharField(choices = MSG_TYPE_CHOICES) + source = models.CharField(max_length=255, blank=True, default='') + scope = models.CharField(choices = SCOPE_CHOICES) + restriction = models.CharField(max_length=255, blank=True, default='') + addresses = models.TextField(blank=True, default='') + code = models.CharField(max_length=255, blank=True, default='') + note = models.TextField(blank=True, default='') + references = models.TextField(blank=True, default='') + incidents = models.TextField(blank=True, default='') def __str__(self): return self.id - # This function is to be used for serialisation + # For serialization def to_dict(self): - dictionary = dict() - dictionary['id'] = self.id - dictionary['identifier'] = self.identifier - dictionary['sender'] = self.sender - dictionary['senderName'] = self.senderName - dictionary['source'] = self.source.url - dictionary['sent'] = str(self.sent) - dictionary['status'] = self.status - dictionary['msg_type'] = self.msg_type - dictionary['scope'] = self.scope - dictionary['urgency'] = self.urgency - dictionary['severity'] = self.severity - dictionary['certainty'] = self.certainty - dictionary['effective'] = str(self.effective) - dictionary['expires'] = str(self.expires) - dictionary['description'] = self.description - dictionary['area_desc'] = self.area_desc - dictionary['event'] = self.event - dictionary['geocode_name'] = self.geocode_name - dictionary['geocode_value'] = self.geocode_value - dictionary['country_id'] = self.country.id - dictionary['country_name'] = self.country.name - - return dictionary - - # To fill uninteresting fields in tests with default values - def set_default_values(self): - self.id = str(timezone.now()) - self.identifier = '' - self.sender = '' - self.senderName = '' - self.source = Source.objects.get(url = "") - self.sent = timezone.now() - self.status = '' - self.msg_type = '' - self.scope = '' - self.urgency = '' - self.severity = '' - self.certainty = '' - self.effective = timezone.now() - self.expires = timezone.now() - self.description = '' - self.area_desc = '' - self.event = '' - self.geocode_name = '' - self.geocode_value = '' - self.country = Country.objects.get(pk = 1) + alert_dict = dict() + alert_dict['country'] = self.country.iso3 + alert_dict['source_feed'] = self.source_feed.url + alert_dict['id'] = self.id + alert_dict['identifier'] = self.identifier + alert_dict['sender'] = self.sender + alert_dict['sent'] = str(self.sent) + alert_dict['status'] = self.status + alert_dict['msg_type'] = self.msg_type + alert_dict['source'] = self.source + alert_dict['scope'] = self.scope + alert_dict['restriction'] = self.restriction + alert_dict['addresses'] = self.addresses + alert_dict['code'] = self.code + alert_dict['note'] = self.note + alert_dict['references'] = self.references + alert_dict['incidents'] = self.incidents + return alert_dict class AlertEncoder(json.JSONEncoder): def default(self, obj): @@ -173,6 +164,86 @@ def default(self, obj): return obj.to_dict() return super().default(obj) + +class AlertInfo(models.Model): + CATEGORY_CHOICES = [ + ('Geo', 'Geo'), + ('Met', 'Met'), + ('Safety', 'Safety'), + ('Security', 'Security'), + ('Rescue', 'Rescue'), + ('Fire', 'Fire'), + ('Health', 'Health'), + ('Env', 'Env'), + ('Transport', 'Transport'), + ('Infra', 'Infra'), + ('CBRNE', 'CBRNE'), + ('Other', 'Other') + ] + + RESPONSE_TYPE_CHOICES = [ + ('Shelter', 'Shelter'), + ('Evacuate', 'Evacuate'), + ('Prepare', 'Prepare'), + ('Execute', 'Execute'), + ('Avoid', 'Avoid'), + ('Monitor', 'Monitor'), + ('Assess', 'Assess'), + ('AllClear', 'AllClear'), + ('None', 'None') + ] + + URGENCY_CHOICES = [ + ('Immediate', 'Immediate'), + ('Expected', 'Expected'), + ('Future', 'Future'), + ('Past', 'Past'), + ('Unknown', 'Unknown') + ] + + SEVERITY_CHOICES = [ + ('Extreme', 'Extreme'), + ('Severe', 'Severe'), + ('Moderate', 'Moderate'), + ('Minor', 'Minor'), + ('Unknown', 'Unknown') + ] + + CERTAINTY_CHOICES = [ + ('Observed', 'Observed'), + ('Likely', 'Likely'), + ('Possible', 'Possible'), + ('Unlikely', 'Unlikely'), + ('Unknown', 'Unknown') + ] + + alert = models.ForeignKey(Alert, on_delete=models.CASCADE) + + language = models.CharField(max_length=255, blank=True, default='en-US') + category = models.CharField(choices = CATEGORY_CHOICES) + event = models.CharField(max_length=255) + response_type = models.CharField(choices = RESPONSE_TYPE_CHOICES, blank=True, default='') + urgency = models.CharField(choices = URGENCY_CHOICES) + severity = models.CharField(choices = SEVERITY_CHOICES) + certainty = models.CharField(choices = CERTAINTY_CHOICES) + audience = models.CharField(blank=True, default='') + event_code = models.CharField(max_length=255, blank=True, default='') + #effective = models.DateTimeField(default=Alert.objects.get(pk=alert).sent) + effective = models.DateTimeField(blank=True, default=timezone.now) + onset = models.DateTimeField(blank=True, null=True) + expires = models.DateTimeField(blank=True, default=(timezone.now() + timedelta(days=1))) + sender_name = models.CharField(max_length=255, blank=True, default='') + headline = models.CharField(max_length=255, blank=True, default='') + description = models.TextField(blank=True, default='') + instruction = models.TextField(blank=True, default='') + web = models.URLField(blank=True, null=True) + contact = models.CharField(max_length=255, blank=True, default='') + parameter = models.CharField(max_length=255, blank=True, default='') + + def __str__(self): + return self.alert.id + ' ' + self.language + + # Adds source to a periodic task def add_source(source): @@ -199,7 +270,7 @@ def add_source(source): ) new_task.save() except Exception as e: - print('crashed lol ', e) + print('Error adding new periodic task', e) # If there is a task with the same interval, add the source to the task else: kwargs = json.loads(existing_task.kwargs) diff --git a/cap_feed/tasks.py b/cap_feed/tasks.py index a79df62..2adfee0 100644 --- a/cap_feed/tasks.py +++ b/cap_feed/tasks.py @@ -1,7 +1,7 @@ from __future__ import absolute_import, unicode_literals from celery import shared_task -import cap_feed.alert_processing as ap +import cap_feed.alert_processor as ap @shared_task(bind=True) def poll_new_alerts(self, sources): diff --git a/cap_feed/templates/cap_feed/index.html b/cap_feed/templates/cap_feed/index.html index 2548f1f..ea432f2 100644 --- a/cap_feed/templates/cap_feed/index.html +++ b/cap_feed/templates/cap_feed/index.html @@ -1,18 +1,18 @@ {% load static %} +

LINK: /admin

+

LINK: /graphql

+
{% if latest_alert_list %}
    {% for alert in latest_alert_list %} -

    {{ alert.area_desc }}

    -

    {{ alert.event }}

    -

    Certainty: {{ alert.certainty }}

    -

    Severity: {{ alert.severity }}

    -

    Urgency: {{ alert.urgency }}

    -

    Sent: {{ alert.sent }}

    -

    Effective: {{ alert.effective }}

    -

    Expires: {{ alert.expires }}

    -

    Sender: {{ alert.sender }}

    -

    Country: {{ alert.country }}

    +

    {{ alert.country }}

    +

    {{ alert.source_feed.url }}

    +

    identifier: {{ alert.identifier }}

    +

    sender: {{ alert.sender }}

    +

    sent: {{ alert.sent }}

    +

    status: {{ alert.status }}

    +

    msg_type: {{ alert.msg_type }}



    {% endfor %}
diff --git a/cap_feed/tests.py b/cap_feed/tests.py index ee208bf..71798e1 100644 --- a/cap_feed/tests.py +++ b/cap_feed/tests.py @@ -1,12 +1,12 @@ import pytz from datetime import datetime -import cap_feed.alert_processing as ap +from cap_feed.formats.utils import convert_datetime +from cap_feed import alert_processor as ap from django.test import TestCase -from django.urls import reverse from django.utils import timezone -from .models import Continent, Region, Country, Source, Alert +from .models import Alert, AlertInfo, Country, Source class AlertModelTests(TestCase): fixtures = ['cap_feed/fixtures/test_data.json'] @@ -17,17 +17,25 @@ def test_alert_source_datetime_converted_to_utc(self): """ cap_sent = "2023-06-24T22:00:00-05:00" cap_effective = "2023-06-24T22:00:00+00:00" + cap_onset = "2023-06-24T22:00:00-00:00" cap_expires = "2023-06-24T22:00:00+05:00" + alert = Alert() - alert.sent = ap.convert_datetime(cap_sent) - alert.effective = ap.convert_datetime(cap_effective) - alert.expires = ap.convert_datetime(cap_expires) + alert.sent = convert_datetime(cap_sent) + alert_info = AlertInfo() + alert_info.effective = convert_datetime(cap_effective) + alert_info.onset = convert_datetime(cap_onset) + alert_info.expires = convert_datetime(cap_expires) + utc_sent = datetime(2023, 6, 25, 3, 0, 0, 0, pytz.UTC) utc_effective = datetime(2023, 6, 24, 22, 0, 0, 0, pytz.UTC) + utc_onset = datetime(2023, 6, 24, 22, 0, 0, 0, pytz.UTC) utc_expires = datetime(2023, 6, 24, 17, 0, 0, 0, pytz.UTC) + assert alert.sent == utc_sent - assert alert.effective == utc_effective - assert alert.expires == utc_expires + assert alert_info.effective == utc_effective + assert alert_info.onset == utc_onset + assert alert_info.expires == utc_expires def test_django_timezone_is_utc(self): """ @@ -41,30 +49,71 @@ def test_expired_alert_is_removed(self): Is an expired alert identified and removed from the database? """ alert = Alert() - alert.set_default_values() - alert.expires = timezone.now() - timezone.timedelta(days = 1) + alert.country = Country.objects.get(pk=1) + alert.source_feed = Source.objects.get(url="") + alert.id = "" + alert.identifier = "" + alert.sender = "" + alert.sent = timezone.now() + alert.status = 'Actual' + alert.msg_type = 'Alert' + alert.scope = 'Public' + + alert_info = AlertInfo() + alert_info.alert = alert + alert_info.category = 'Met' + alert_info.event = '' + alert_info.urgency = 'Immediate' + alert_info.severity = 'Extreme' + alert_info.certainty = 'Observed' + alert_info.expires = timezone.now() - timezone.timedelta(days = 1) + try: alert.save() + alert_info.save() # catch redis connection errors, not relevant for this test except ValueError: pass - previous_count = Alert.objects.count() + + previous_alert_count = Alert.objects.count() + previous_alert_info_count = AlertInfo.objects.count() ap.remove_expired_alerts() - assert Alert.objects.count() == previous_count - 1 + assert Alert.objects.count() == previous_alert_count - 1 + assert AlertInfo.objects.count() == previous_alert_info_count - 1 def test_unexpired_alert_is_not_removed(self): """ - Is an unexpired alert kept in the database? + Is an expired alert identified and removed from the database? """ alert = Alert() - alert.set_default_values() - alert.expires = timezone.now() + timezone.timedelta(days = 1) + alert.country = Country.objects.get(pk=1) + alert.source_feed = Source.objects.get(url="") + alert.id = "" + alert.identifier = "" + alert.sender = "" + alert.sent = timezone.now() + alert.status = 'Actual' + alert.msg_type = 'Alert' + alert.scope = 'Public' + + alert_info = AlertInfo() + alert_info.alert = alert + alert_info.category = 'Met' + alert_info.event = '' + alert_info.urgency = 'Immediate' + alert_info.severity = 'Extreme' + alert_info.certainty = 'Observed' + alert_info.expires = timezone.now() + timezone.timedelta(days = 1) + try: alert.save() + alert_info.save() # catch redis connection errors, not relevant for this test except ValueError: pass - previous_count = Alert.objects.count() + + previous_alert_count = Alert.objects.count() + previous_alert_info_count = AlertInfo.objects.count() ap.remove_expired_alerts() - assert Alert.objects.count() == previous_count - alert.delete() \ No newline at end of file + assert Alert.objects.count() == previous_alert_count + assert AlertInfo.objects.count() == previous_alert_info_count \ No newline at end of file diff --git a/cap_feed/views.py b/cap_feed/views.py index ab3d83a..d7e0890 100644 --- a/cap_feed/views.py +++ b/cap_feed/views.py @@ -1,26 +1,22 @@ -import json -import cap_feed.alert_processing as ap - +import cap_feed.data_injector as dl from django.http import HttpResponse -from django.utils import timezone from django.template import loader -from django_celery_beat.models import IntervalSchedule, PeriodicTask -from django_celery_beat.models import PeriodicTask -from .models import Alert, Source, SourceEncoder +from .models import Alert, Source + +import cap_feed.alert_processor as ap def index(request): try: - ap.inject_geographical_data() - except Exception as e: - print(e) - return HttpResponse(f"Error while injecting geographical data {e}") - try: - ap.inject_sources() + dl.inject_geographical_data() + if Source.objects.count() == 0: + dl.inject_sources() + #ap.remove_expired_alerts() + #ap.poll_new_alerts([]) except Exception as e: print(e) - return HttpResponse(f"Error while injecting source data {e}") + return HttpResponse(f"Error while injecting data {e}") latest_alert_list = Alert.objects.order_by("-sent")[:10] template = loader.get_template("cap_feed/index.html") diff --git a/capaggregator/schema.py b/capaggregator/schema.py index 1d186c7..e9d45e5 100644 --- a/capaggregator/schema.py +++ b/capaggregator/schema.py @@ -1,9 +1,13 @@ import graphene from graphene_django import DjangoObjectType -from cap_feed.models import Continent, Region, Country, Alert +from cap_feed.models import Continent, Region, Country, Alert, AlertInfo +class AlertInfoType(DjangoObjectType): + class Meta: + model = AlertInfo + class AlertType(DjangoObjectType): class Meta: model = Alert @@ -23,6 +27,7 @@ class Meta: class Query(graphene.ObjectType): list_alert=graphene.List(AlertType, iso3=graphene.String(), region_id=graphene.String(), continent_id=graphene.String()) + list_alert_info=graphene.List(AlertInfoType, iso3=graphene.String(), region_id=graphene.String(), continent_id=graphene.String()) list_continent=graphene.List(ContinentType) list_country=graphene.List(CountryType, region_id=graphene.String(), continent_id=graphene.String()) list_region=graphene.List(RegionType) @@ -40,6 +45,19 @@ def resolve_list_alert(root, info, iso3=None, region_id=None, continent_id=None, return Alert.objects.all() + def resolve_list_alert_info(root, info, iso3=None, region_id=None, continent_id=None, **kwargs): + filter = dict() + if iso3: + filter['alert__country__iso3'] = iso3 + if region_id: + filter['alert__country__region'] = region_id + if continent_id: + filter['alert__country__continent'] = continent_id + if len(filter) > 0: + return AlertInfo.objects.filter(**filter).all() + + return AlertInfo.objects.all() + def resolve_list_continent(root, info): return Continent.objects.all() diff --git a/capaggregator/settings.py b/capaggregator/settings.py index 2cd70a3..8235455 100644 --- a/capaggregator/settings.py +++ b/capaggregator/settings.py @@ -34,7 +34,7 @@ # Application definition INSTALLED_APPS = [ - 'alert_subscription.apps.AlertSubscriptionConfig', + #'alert_subscription.apps.AlertSubscriptionConfig', 'daphne', 'channels', 'django_celery_results',