Update docker-compose.yml for production

.github/workflows/ci.yml

@@ -28,7 +28,7 @@ jobs:
 
           # Share PA_BRANCH variable amount steps
           echo "::set-env name=ID_BRANCH_NAME::$ID_BRANCH_NAME"
-          touch .env
+          cp docker-sample.env docker.env
 
       - name: Docker pull
         run: docker-compose pull localdb api || ID_BRANCH_NAME= docker-compose pull localdb api || true

.gitignore

@@ -25,3 +25,5 @@ source/supervisord.log
 source/supervisord.pid
 .vscode/settings.json
 .env
+
+docker.env

docker-compose.yml

@@ -24,7 +24,7 @@ services:
       # IMPORTANT: Remove this on production
       - POSTGRES_HOST_AUTH_METHOD=trust
     volumes:
-      - ./pgdata:/var/lib/postgresql/data
+      - postgres-data:/var/lib/postgresql/data
     ports:
       - 127.0.0.1:5433:5432
 
@@ -66,3 +66,7 @@ services:
     command: sh -c './scripts/wait-for-it.sh localdb:5432 && python3 -m unittest /home/idetect/python/idetect/tests/test_*.py'
     volumes:
       - ./source:/home/idetect
+
+
+volumes:
+  postgres-data:

docker-prod.yml

@@ -1,22 +1,21 @@
 version: '3.4'
 
 x-base-idetect-setup: &x-base-idetect-setup
-  image: docker.pkg.github.com/idmc-labs/idetect/idetect:$RELEASE
+  image: docker.pkg.github.com/idmc-labs/idetect/idetect:${RELEASE-release}
+  build:
+    context: .
+    dockerfile: ./docker/idetect/Dockerfile
+    cache_from:
+      - "docker.pkg.github.com/idmc-labs/idetect/idetect:${RELEASE}"
+      - docker.pkg.github.com/idmc-labs/idetect/idetect:release
+      - docker.pkg.github.com/idmc-labs/idetect/idetect:latest
+    args:
+      - UID=${UID}
   env_file: docker.env
   user: ${UID}
-  depends_on:
-    - localdb
-
+  restart: on-failure
 
 services:
-
-  localdb:
-    image: docker.pkg.github.com/idmc-labs/idetect/localdb:$RELEASE
-    ports:
-      - 127.0.0.1:5433:5432
-    volumes:
-      - ./pgdata:/var/lib/postgresql/data
-
   api:
     <<: *x-base-idetect-setup
     command: sh -c "./scripts/wait-for-it.sh localdb:5432 && /usr/bin/supervisord -c /idetect-configs/api-supervisord.conf"
@@ -29,13 +28,13 @@ services:
     volumes:
       - ./logs:/var/log
 
-  notebooks:
-    <<: *x-base-idetect-setup
-    command: jupyter notebook --no-browser --ip=0.0.0.0 --port 5002 /home/idetect/notebooks
-    ports:
-      - 5002:5002
-
   cron:
     <<: *x-base-idetect-setup
     user: root
     command: sh -c 'cron -f -L 7 /idetect-configs/crontab'
+
+  # notebooks:
+  #   <<: *x-base-idetect-setup
+  #   command: jupyter notebook --no-browser --ip=0.0.0.0 --port 5002 /home/idetect/notebooks
+  #   ports:
+  #     - 5002:5002

docker.env → docker-sample.env

@@ -1,11 +1,11 @@
 # These DB connection parameters are set up in docker/postgres/docker-entrypoint-initdb.d/project.sh
 DB_HOST=localdb
 DB_USER=idetect
-DB_PASS=democracy
+DB_PASSWORD=democracy
 DB_NAME=idetect
 DB_PORT=5432
 
 # Python packages in source/python are made available to python-based containers here
 PYTHONPATH=/home/idetect/python
 
-MAPZEN_KEY=thisisnotakey
+MAPZEN_KEY=thisisnotakey

docker/idetect/Dockerfile

@@ -1,11 +1,11 @@
-FROM ubuntu:latest
+FROM ubuntu:18.04
 
 ENV DEBIAN_FRONTEND noninteractive
 
 # python basics
 RUN apt-get clean && \
     apt-get update && \
-    apt-get -y install python3 python3-pip python3-dev
+    apt-get -y install python3.6 python3-pip python3-dev
 
 ## download the spacy model using curl for progress indication
 # This is quite large, so we're downloading it early in the build for the sake of caching

source/python/idetect/model.py

@@ -31,7 +31,7 @@ def db_url():
         user=os.environ.get('DB_USER'),
         passwd=os.environ.get('DB_PASSWORD'),
         db_host=os.environ.get('DB_HOST'),
-        db_port=os.environ.get('DB_PORT'),
+        db_port=os.environ.get('DB_PORT', 5432),
         db=os.environ.get('DB_NAME'))
 
 

source/scripts/update_mviews.bash

@@ -1,5 +1,15 @@
 #!/bin/bash
 
-echo "REFRESH MATERIALIZED VIEW idetect_fact_api_locations" | psql -h localdb -U idetect
-echo "REFRESH MATERIALIZED VIEW idetect_fact_api" | psql -h localdb -U idetect
-echo "REFRESH MATERIALIZED VIEW idetect_map_week_mview" | psql -h localdb -U idetect
+# Environment variables required by psql
+export PGUSER=${DB_USER}
+export PGPASSWORD=${DB_PASSWORD}
+export PGDATABASE=${DB_NAME}
+export PGHOST=${DB_HOST}
+export PGPORT=${DB_PORT}
+
+set -x
+psql -c "REFRESH MATERIALIZED VIEW idetect_map_mview;"
+psql -c "REFRESH MATERIALIZED VIEW idetect_map_week_mview;"
+psql -c "REFRESH MATERIALIZED VIEW idetect_fact_api_locations"
+psql -c "REFRESH MATERIALIZED VIEW idetect_fact_api"
+set +x