Fetching starting kit repos via HTTP + fix CI (#592)

.pre-commit-config.yaml

@@ -19,4 +19,4 @@ repos:
     hooks:
     -   id: mypy
         types: [file, python]
-        additional_dependencies: [types-PyYAML, types-click, types-itsdangerous, types-Flask, types-Werkzeug]
+        additional_dependencies: [types-PyYAML, types-click, types-itsdangerous, types-Flask, types-Werkzeug, types-requests]

ci_tools/circle/build_doc.sh

@@ -4,14 +4,14 @@ set -e
 
 # Install dependencies with miforge
 curl -L -O https://github.com/conda-forge/miniforge/releases/latest/download/Miniforge3-$(uname)-$(uname -m).sh
-bash Miniforge3-$(uname)-$(uname -m).sh -b -p $MINIFORGE_PATH
-export PATH="$MINIFORGE_PATH/bin:$PATH"
+bash Miniforge3-$(uname)-$(uname -m).sh -b -p /home/circleci/project/miniforge/
+export PATH="/home/circleci/project/miniforge/bin:$PATH"
 
 # create the environment
 conda create --yes -n testenv python=3.8
 conda env update -n testenv -f environment.yml
 source activate testenv
-conda install --yes sphinx=3.5.4 jinja2=3.0.3 sphinx_rtd_theme numpydoc pygraphviz
+conda install --yes sphinx=3.5.4 jinja2=3.0.3 sphinx_rtd_theme numpydoc==1.2.1 pygraphviz
 pip install eralchemy sphinx-click
 
 # Build and install scikit-learn in dev mode

doc/whats_new/v0.11.rst

@@ -0,0 +1,13 @@
+.. _changes_0_10:
+
+Version 0.11
+============
+
+Changelog
+---------
+
+
+`ramp-database`
+...............
+
+- Switch to fetching starting kit repos via HTTP rather than using git clone to avoid being blocked by Github :pr:`592`.

ramp-database/ramp_database/testing.py

@@ -7,8 +7,9 @@
 import os
 import shutil
 import subprocess
-
-from git import Repo
+import requests
+from io import BytesIO
+from zipfile import ZipFile
 
 from ramp_utils import read_config
 from ramp_utils import generate_ramp_config
@@ -123,6 +124,28 @@ def _delete_line_from_file(f_name, line_to_delete):
         f.truncate()
 
 
+def _fetch_github_repo(url, output_dir):
+    """Fetch a zip of the github repo"""
+    response = requests.get(url)
+    if response.status_code != 200:
+        raise ValueError(
+            f"Failed to download the archive. HTTP status code:"
+            f"{response.status_code}"
+        )
+
+    with ZipFile(BytesIO(response.content)) as fh:
+        for member in fh.namelist():
+            # Skip directories
+            if member.endswith("/"):
+                continue
+            # there is an extra root folder f"{problem-name}-master" in the zip
+            # we want to remove when extracing
+            target_path = Path(output_dir) / "/".join(member.split("/")[1:])
+            target_path.parent.mkdir(exist_ok=True, parents=True)
+            with fh.open(member) as source, open(target_path, "wb") as target:
+                shutil.copyfileobj(source, target)
+
+
 def setup_ramp_kit_ramp_data(
     ramp_config,
     problem_name,
@@ -158,11 +181,11 @@ def setup_ramp_kit_ramp_data(
                 'it, you need to set "force=True".'
             )
         shutil.rmtree(problem_kit_path, ignore_errors=True)
-    ramp_kit_url = "https://github.com/ramp-kits/{}.git".format(problem_name)
-    kwargs = {}
-    if depth is not None:
-        kwargs["depth"] = depth
-    Repo.clone_from(ramp_kit_url, problem_kit_path, **kwargs)
+
+    ramp_kit_url = (
+        f"https://github.com/ramp-kits/{problem_name}/archive/refs/heads/master.zip"
+    )
+    _fetch_github_repo(ramp_kit_url, problem_kit_path)
 
     problem_data_path = ramp_config["ramp_data_dir"]
     if os.path.exists(problem_data_path):
@@ -172,8 +195,10 @@ def setup_ramp_kit_ramp_data(
                 'it, you need to set "force=True".'
             )
         shutil.rmtree(problem_data_path, ignore_errors=True)
-    ramp_data_url = "https://github.com/ramp-data/{}.git".format(problem_name)
-    Repo.clone_from(ramp_data_url, problem_data_path, **kwargs)
+    ramp_data_url = (
+        f"https://github.com/ramp-data/{problem_name}/archive/refs/heads/master.zip"
+    )
+    _fetch_github_repo(ramp_data_url, problem_data_path)
 
     current_directory = os.getcwd()
     os.chdir(problem_data_path)

requirements.txt

@@ -11,7 +11,6 @@ Flask-Login==0.5.0
 Flask-Mail==0.9.1
 Flask-SQLAlchemy==2.5.1
 Flask-Wtf==1.0.0
-gitpython
 ipykernel
 jupyter
 numpy