From a92443948cfe1851d2dd55e73db036a2f20f6779 Mon Sep 17 00:00:00 2001
From: Roman Yurchak <rth.yurchak@gmail.com>
Date: Fri, 1 Mar 2024 11:18:25 +0100
Subject: [PATCH] Fetching starting kit repos via HTTP + fix CI (#592)

---
 .pre-commit-config.yaml                |  2 +-
 ci_tools/circle/build_doc.sh           |  6 ++--
 doc/whats_new/v0.11.rst                | 13 ++++++++
 ramp-database/ramp_database/testing.py | 43 ++++++++++++++++++++------
 requirements.txt                       |  1 -
 5 files changed, 51 insertions(+), 14 deletions(-)
 create mode 100644 doc/whats_new/v0.11.rst

diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml
index 5e6e9248..7f29f65a 100644
--- a/.pre-commit-config.yaml
+++ b/.pre-commit-config.yaml
@@ -19,4 +19,4 @@ repos:
     hooks:
     -   id: mypy
         types: [file, python]
-        additional_dependencies: [types-PyYAML, types-click, types-itsdangerous, types-Flask, types-Werkzeug]
+        additional_dependencies: [types-PyYAML, types-click, types-itsdangerous, types-Flask, types-Werkzeug, types-requests]
diff --git a/ci_tools/circle/build_doc.sh b/ci_tools/circle/build_doc.sh
index 5f4f7810..72e32c84 100755
--- a/ci_tools/circle/build_doc.sh
+++ b/ci_tools/circle/build_doc.sh
@@ -4,14 +4,14 @@ set -e
 
 # Install dependencies with miforge
 curl -L -O https://github.com/conda-forge/miniforge/releases/latest/download/Miniforge3-$(uname)-$(uname -m).sh
-bash Miniforge3-$(uname)-$(uname -m).sh -b -p $MINIFORGE_PATH
-export PATH="$MINIFORGE_PATH/bin:$PATH"
+bash Miniforge3-$(uname)-$(uname -m).sh -b -p /home/circleci/project/miniforge/
+export PATH="/home/circleci/project/miniforge/bin:$PATH"
 
 # create the environment
 conda create --yes -n testenv python=3.8
 conda env update -n testenv -f environment.yml
 source activate testenv
-conda install --yes sphinx=3.5.4 jinja2=3.0.3 sphinx_rtd_theme numpydoc pygraphviz
+conda install --yes sphinx=3.5.4 jinja2=3.0.3 sphinx_rtd_theme numpydoc==1.2.1 pygraphviz
 pip install eralchemy sphinx-click
 
 # Build and install scikit-learn in dev mode
diff --git a/doc/whats_new/v0.11.rst b/doc/whats_new/v0.11.rst
new file mode 100644
index 00000000..26608510
--- /dev/null
+++ b/doc/whats_new/v0.11.rst
@@ -0,0 +1,13 @@
+.. _changes_0_10:
+
+Version 0.11
+============
+
+Changelog
+---------
+
+
+`ramp-database`
+...............
+
+- Switch to fetching starting kit repos via HTTP rather than using git clone to avoid being blocked by Github :pr:`592`.
diff --git a/ramp-database/ramp_database/testing.py b/ramp-database/ramp_database/testing.py
index 717a834e..f177de5c 100644
--- a/ramp-database/ramp_database/testing.py
+++ b/ramp-database/ramp_database/testing.py
@@ -7,8 +7,9 @@
 import os
 import shutil
 import subprocess
-
-from git import Repo
+import requests
+from io import BytesIO
+from zipfile import ZipFile
 
 from ramp_utils import read_config
 from ramp_utils import generate_ramp_config
@@ -123,6 +124,28 @@ def _delete_line_from_file(f_name, line_to_delete):
         f.truncate()
 
 
+def _fetch_github_repo(url, output_dir):
+    """Fetch a zip of the github repo"""
+    response = requests.get(url)
+    if response.status_code != 200:
+        raise ValueError(
+            f"Failed to download the archive. HTTP status code:"
+            f"{response.status_code}"
+        )
+
+    with ZipFile(BytesIO(response.content)) as fh:
+        for member in fh.namelist():
+            # Skip directories
+            if member.endswith("/"):
+                continue
+            # there is an extra root folder f"{problem-name}-master" in the zip
+            # we want to remove when extracing
+            target_path = Path(output_dir) / "/".join(member.split("/")[1:])
+            target_path.parent.mkdir(exist_ok=True, parents=True)
+            with fh.open(member) as source, open(target_path, "wb") as target:
+                shutil.copyfileobj(source, target)
+
+
 def setup_ramp_kit_ramp_data(
     ramp_config,
     problem_name,
@@ -158,11 +181,11 @@ def setup_ramp_kit_ramp_data(
                 'it, you need to set "force=True".'
             )
         shutil.rmtree(problem_kit_path, ignore_errors=True)
-    ramp_kit_url = "https://github.com/ramp-kits/{}.git".format(problem_name)
-    kwargs = {}
-    if depth is not None:
-        kwargs["depth"] = depth
-    Repo.clone_from(ramp_kit_url, problem_kit_path, **kwargs)
+
+    ramp_kit_url = (
+        f"https://github.com/ramp-kits/{problem_name}/archive/refs/heads/master.zip"
+    )
+    _fetch_github_repo(ramp_kit_url, problem_kit_path)
 
     problem_data_path = ramp_config["ramp_data_dir"]
     if os.path.exists(problem_data_path):
@@ -172,8 +195,10 @@ def setup_ramp_kit_ramp_data(
                 'it, you need to set "force=True".'
             )
         shutil.rmtree(problem_data_path, ignore_errors=True)
-    ramp_data_url = "https://github.com/ramp-data/{}.git".format(problem_name)
-    Repo.clone_from(ramp_data_url, problem_data_path, **kwargs)
+    ramp_data_url = (
+        f"https://github.com/ramp-data/{problem_name}/archive/refs/heads/master.zip"
+    )
+    _fetch_github_repo(ramp_data_url, problem_data_path)
 
     current_directory = os.getcwd()
     os.chdir(problem_data_path)
diff --git a/requirements.txt b/requirements.txt
index 0f151d68..ab1cf3d4 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -11,7 +11,6 @@ Flask-Login==0.5.0
 Flask-Mail==0.9.1
 Flask-SQLAlchemy==2.5.1
 Flask-Wtf==1.0.0
-gitpython
 ipykernel
 jupyter
 numpy