Merge branch '39-readthedocs' into 'develop'

Release 4.0.0-alpha Closes #1, #9, #25, #33, and #39 See merge request remi.cresson/otbtf!74
remicres · Apr 4, 2023 · ed255a5 · ed255a5
2 parents c27887e + 4542a4d
commit ed255a5
Show file tree

Hide file tree

Showing 59 changed files with 4,180 additions and 1,919 deletions.
diff --git a/.gitignore b/.gitignore
@@ -1,3 +1,6 @@
 # Compiled python source #
 *.pyc
 .idea
+build
+site
+otbtf.egg-info
diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
@@ -1,7 +1,7 @@
 image: $CI_REGISTRY_IMAGE:$CI_COMMIT_REF_NAME
 
 variables:
-  OTBTF_VERSION: 3.4.0
+  OTBTF_VERSION: 4.0.0
   OTB_BUILD: /src/otb/build/OTB/build  # Local OTB build directory
   OTBTF_SRC: /src/otbtf  # Local OTBTF source directory
   OTB_TEST_DIR: $OTB_BUILD/Testing/Temporary  # OTB testing directory
@@ -18,14 +18,17 @@ variables:
   CI_REGISTRY_PUBIMG: $CI_REGISTRY_IMAGE:$OTBTF_VERSION
   DOCKERHUB_BASE: mdl4eo/otbtf
   DOCKERHUB_IMAGE_BASE: ${DOCKERHUB_BASE}:${OTBTF_VERSION}
-
+  CPU_BASE_IMG: ubuntu:22.04
+  GPU_BASE_IMG: nvidia/cuda:12.1.0-devel-ubuntu22.04
+
 workflow:
   rules:
     - if: $CI_MERGE_REQUEST_ID || $CI_COMMIT_REF_NAME =~ /master/ # Execute jobs in merge request context, or commit in master branch
 
 stages:
   - Build
   - Static Analysis
+  - Documentation
   - Test
   - Applications Test
   - Update dev image
@@ -51,40 +54,39 @@ docker image:
     - >
       docker build
       --target otbtf-base
-      --network="host"
       --cache-from $CACHE_IMAGE_BASE
       --tag $CACHE_IMAGE_BASE
-      --build-arg BASE_IMG="ubuntu:20.04"
+      --build-arg BASE_IMG=$CPU_BASE_IMG
       --build-arg BUILDKIT_INLINE_CACHE=1
       "."
     - docker push $CACHE_IMAGE_BASE
     - >
       docker build
       --target builder
-      --network="host"
       --cache-from $CACHE_IMAGE_BASE
       --cache-from $CACHE_IMAGE_BUILDER
       --tag $CACHE_IMAGE_BUILDER
-      --build-arg OTBTESTS="true"
       --build-arg KEEP_SRC_OTB="true"
       --build-arg BZL_CONFIGS=""
-      --build-arg BASE_IMG="ubuntu:20.04"
+      --build-arg BASE_IMG=$CPU_BASE_IMG
       --build-arg BUILDKIT_INLINE_CACHE=1
+      --build-arg BZL_OPTIONS="--verbose_failures --remote_cache=$BAZELCACHE"
+      --build-arg OTBTESTS="true"
       "."
     - docker push $CACHE_IMAGE_BUILDER
     - >
       docker build
-      --network="host"
       --cache-from $CACHE_IMAGE_BASE
       --cache-from $CACHE_IMAGE_BUILDER
       --cache-from $BRANCH_IMAGE
       --cache-from $DEV_IMAGE
       --tag $BRANCH_IMAGE
-      --build-arg OTBTESTS="true"
       --build-arg KEEP_SRC_OTB="true"
       --build-arg BZL_CONFIGS=""
-      --build-arg BASE_IMG="ubuntu:20.04"
+      --build-arg BASE_IMG=$CPU_BASE_IMG
       --build-arg BUILDKIT_INLINE_CACHE=1
+      --build-arg BZL_OPTIONS="--verbose_failures --remote_cache=$BAZELCACHE"
+      --build-arg OTBTESTS="true"
       "."
     - docker push $BRANCH_IMAGE
 
@@ -95,26 +97,54 @@ docker image:
 flake8:
   extends: .static_analysis_base
   script:
-    - sudo apt update && sudo apt install flake8 -y
-    - python -m flake8 --max-line-length=120 --per-file-ignores="__init__.py:F401" $OTBTF_SRC/otbtf
+    - sudo pip install flake8
+    - flake8 $OTBTF_SRC/otbtf --exclude=tensorflow_v1x
 
 pylint:
   extends: .static_analysis_base
   script:
-    - sudo apt update && sudo apt install pylint -y
-    - pylint --logging-format-style=old --disable=too-many-nested-blocks,too-many-locals,too-many-statements,too-few-public-methods,too-many-instance-attributes,too-many-arguments --ignored-modules=tensorflow --max-line-length=120 --logging-format-style=new $OTBTF_SRC/otbtf
+    - sudo pip install pylint
+    - pylint $OTBTF_SRC/otbtf --ignore=tensorflow_v1x
 
 codespell:
   extends: .static_analysis_base
   script:
-    - sudo pip install codespell && codespell
-
+    - sudo pip install codespell
+    - codespell otbtf
+    - codespell doc
+
 cppcheck:
   extends: .static_analysis_base
   script:
     - sudo apt update && sudo apt install cppcheck -y
     - cd $OTBTF_SRC/ && cppcheck --enable=all --error-exitcode=1 -I include/ --suppress=missingInclude --suppress=unusedFunction .
 
+.doc_base:
+  stage: Documentation
+  before_script:
+    - pip install -r doc/doc_requirements.txt
+  artifacts:
+    paths:
+      - public
+      - public_test
+
+pages_test:
+  extends: .doc_base
+  except:
+    - master
+  script:
+    - mkdocs build --site-dir public_test
+
+pages:
+  extends: .doc_base
+  only:
+    - master
+  script:
+    - mkdocs build --site-dir public
+  artifacts:
+    paths:
+      - public
+
 .tests_base:
   artifacts:
     paths:
@@ -172,6 +202,11 @@ geos_enabled:
   script:
     - python -m pytest --junitxml=$ARTIFACT_TEST_DIR/report_geos_enabled.xml $OTBTF_SRC/test/geos_test.py
 
+imports:
+  extends: .applications_test_base
+  script:
+    - python -m pytest --junitxml=$ARTIFACT_TEST_DIR/report_imports.xml $OTBTF_SRC/test/imports_test.py
+
 deploy_cpu-dev-testing:
   stage: Update dev image
   extends: .docker_build_base
@@ -198,10 +233,10 @@ deploy_cpu:
     DOCKERHUB_LATEST: $DOCKERHUB_BASE:latest
   script:
     # cpu
-    - docker build --network='host' --tag $IMAGE_CPU --build-arg BASE_IMG=ubuntu:20.04 --build-arg BZL_CONFIGS="" .
+    - docker build --build-arg BZL_OPTIONS="--remote_cache=$BAZELCACHE" --tag $IMAGE_CPU --build-arg BASE_IMG=$CPU_BASE_IMG --build-arg BZL_CONFIGS="" .
     - docker push $IMAGE_CPU
     # cpu-dev
-    - docker build --network='host' --tag $IMAGE_CPUDEV --build-arg BASE_IMG=ubuntu:20.04 --build-arg BZL_CONFIGS="" --build-arg KEEP_SRC_OTB=true .
+    - docker build --build-arg BZL_OPTIONS="--remote_cache=$BAZELCACHE" --tag $IMAGE_CPUDEV --build-arg BASE_IMG=$CPU_BASE_IMG --build-arg BZL_CONFIGS="" --build-arg KEEP_SRC_OTB=true .
     - docker push $IMAGE_CPUDEV
     # push images on dockerhub
     - echo -n $DOCKERHUB_TOKEN | docker login -u mdl4eo --password-stdin
@@ -224,16 +259,16 @@ deploy_gpu:
     DOCKERHUB_GPUDEV: $DOCKERHUB_IMAGE_BASE-gpu-dev
   script:
     # gpu-opt
-    - docker build --network='host' --tag $IMAGE_GPUOPT --build-arg BASE_IMG=nvidia/cuda:11.2.2-cudnn8-devel-ubuntu20.04 .
+    - docker build --build-arg BZL_OPTIONS="--remote_cache=$BAZELCACHE" --tag $IMAGE_GPUOPT --build-arg BASE_IMG=$GPU_BASE_IMG .
     - docker push $IMAGE_GPUOPT
     # gpu-opt-dev
-    - docker build --network='host' --tag $IMAGE_GPUOPTDEV --build-arg BASE_IMG=nvidia/cuda:11.2.2-cudnn8-devel-ubuntu20.04 --build-arg KEEP_SRC_OTB=true .
+    - docker build --build-arg BZL_OPTIONS="--remote_cache=$BAZELCACHE" --tag $IMAGE_GPUOPTDEV --build-arg BASE_IMG=$GPU_BASE_IMG --build-arg KEEP_SRC_OTB=true .
     - docker push $IMAGE_GPUOPTDEV
     # gpu-basic
-    - docker build --network='host' --tag $IMAGE_GPU --build-arg BASE_IMG=nvidia/cuda:11.2.2-cudnn8-devel-ubuntu20.04 --build-arg BZL_CONFIGS="" .
+    - docker build --build-arg BZL_OPTIONS="--remote_cache=$BAZELCACHE" --tag $IMAGE_GPU --build-arg BASE_IMG=$GPU_BASE_IMG --build-arg BZL_CONFIGS="" .
     - docker push $IMAGE_GPU
     # gpu-basic-dev
-    - docker build --network='host' --tag $IMAGE_GPUDEV --build-arg BZL_CONFIGS="" --build-arg BASE_IMG=nvidia/cuda:11.2.2-cudnn8-devel-ubuntu20.04 --build-arg KEEP_SRC_OTB=true .
+    - docker build --build-arg BZL_OPTIONS="--remote_cache=$BAZELCACHE" --tag $IMAGE_GPUDEV --build-arg BZL_CONFIGS="" --build-arg BASE_IMG=$GPU_BASE_IMG --build-arg KEEP_SRC_OTB=true .
     - docker push $IMAGE_GPUDEV
     # push gpu-basic* images on dockerhub
     - echo -n $DOCKERHUB_TOKEN | docker login -u mdl4eo --password-stdin

diff --git a/.readthedocs.yaml b/.readthedocs.yaml
@@ -0,0 +1,12 @@
+version: 2
+build:
+  os: ubuntu-22.04
+  tools:
+    python: "3.10"
+
+mkdocs:
+  configuration: mkdocs.yml
+
+python:
+   install:
+   - requirements: doc/doc_requirements.txt
diff --git a/Dockerfile b/Dockerfile
@@ -28,7 +28,7 @@ RUN pip install --no-cache-dir pip --upgrade
 # NumPy version is conflicting with system's gdal dep and may require venv
 ARG NUMPY_SPEC="==1.22.*"
 ARG PROTO_SPEC="==3.20.*"
-RUN pip install --no-cache-dir -U wheel mock six future tqdm deprecated "numpy$NUMPY_SPEC" "protobuf$PROTO_SPEC" \
+RUN pip install --no-cache-dir -U wheel mock six future tqdm deprecated "numpy$NUMPY_SPEC" "protobuf$PROTO_SPEC" packaging requests \
  && pip install --no-cache-dir --no-deps keras_applications keras_preprocessing
 
 # ----------------------------------------------------------------------------
@@ -37,44 +37,52 @@ FROM otbtf-base AS builder
 # A smaller value may be required to avoid OOM errors when building OTB GUI
 ARG CPU_RATIO=1
 
-RUN mkdir -p /src/tf /opt/otbtf/bin /opt/otbtf/include /opt/otbtf/lib
+RUN mkdir -p /src/tf /opt/otbtf/bin /opt/otbtf/include /opt/otbtf/lib/python3
 WORKDIR /src/tf
 
 RUN git config --global advice.detachedHead false
 
 ### TF
-ARG TF=v2.8.0
+
+ARG TF=v2.12.0
+
 # Install bazelisk (will read .bazelversion and download the right bazel binary - latest by default)
 RUN wget -qO /opt/otbtf/bin/bazelisk https://github.com/bazelbuild/bazelisk/releases/latest/download/bazelisk-linux-amd64 \
  && chmod +x /opt/otbtf/bin/bazelisk \
  && ln -s /opt/otbtf/bin/bazelisk /opt/otbtf/bin/bazel
 
 ARG BZL_TARGETS="//tensorflow:libtensorflow_cc.so //tensorflow/tools/pip_package:build_pip_package"
-# "--config=opt" will enable 'march=native' (otherwise read comments about CPU compatibility and edit CC_OPT_FLAGS in build-env-tf.sh)
+
+# "--config=opt" will enable 'march=native'
+# (otherwise read comments about CPU compatibility and edit CC_OPT_FLAGS in
+# build-env-tf.sh)
 ARG BZL_CONFIGS="--config=nogcp --config=noaws --config=nohdfs --config=opt"
-# "--compilation_mode opt" is already enabled by default (see tf repo .bazelrc and configure.py)
+
+# "--compilation_mode opt" is already enabled by default (see tf repo .bazelrc
+# and configure.py)
 ARG BZL_OPTIONS="--verbose_failures --remote_cache=http://localhost:9090"
 
 # Build
 ARG ZIP_TF_BIN=false
 COPY tools/docker/build-env-tf.sh ./
-RUN git clone --single-branch -b $TF https://github.com/tensorflow/tensorflow.git \
- && cd tensorflow \
+RUN git clone --single-branch -b $TF https://github.com/tensorflow/tensorflow.git
+RUN cd tensorflow \
  && export PATH=$PATH:/opt/otbtf/bin \
  && export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:/opt/otbtf/lib \
  && bash -c '\
       source ../build-env-tf.sh \
       && ./configure \
       && export TMP=/tmp/bazel \
       && BZL_CMD="build $BZL_TARGETS $BZL_CONFIGS $BZL_OPTIONS" \
-      && bazel $BZL_CMD --jobs="HOST_CPUS*$CPU_RATIO" ' \
-# Installation - split here if you want to check files  ^
-#RUN cd tensorflow \
+      && bazel $BZL_CMD --jobs="HOST_CPUS*$CPU_RATIO" '
+
+# Installation
+RUN cd tensorflow \
  && ./bazel-bin/tensorflow/tools/pip_package/build_pip_package /tmp/tensorflow_pkg \
  && pip3 install --no-cache-dir --prefix=/opt/otbtf /tmp/tensorflow_pkg/tensorflow*.whl \
- && ln -s /opt/otbtf/lib/python3.* /opt/otbtf/lib/python3 \
- && cp -P bazel-bin/tensorflow/libtensorflow_cc.so* /opt/otbtf/lib/ \
- && ln -s $(find /opt/otbtf -type d -wholename "*/site-packages/tensorflow/include") /opt/otbtf/include/tf \
+ && ln -s /opt/otbtf/local/lib/python3.*/* /opt/otbtf/lib/python3 \
+ && ln -s /opt/otbtf/local/bin/* /opt/otbtf/bin \
+ && ln -s $(find /opt/otbtf -type d -wholename "*/dist-packages/tensorflow/include") /opt/otbtf/include/tf \
  # The only missing header in the wheel
  && cp tensorflow/cc/saved_model/tag_constants.h /opt/otbtf/include/tf/tensorflow/cc/saved_model/ \
  && cp tensorflow/cc/saved_model/signature_constants.h /opt/otbtf/include/tf/tensorflow/cc/saved_model/ \
@@ -86,6 +94,7 @@ RUN git clone --single-branch -b $TF https://github.com/tensorflow/tensorflow.gi
  && rm -rf bazel-* /src/tf /root/.cache/ /tmp/*
 
 ### OTB
+
 ARG GUI=false
 ARG OTB=5086d7601d80f2427f4d4d7f2398ec46e7efa300
 ARG OTBTESTS=false
@@ -99,7 +108,20 @@ RUN apt-get update -y \
  && apt-get install --reinstall ca-certificates -y \
  && update-ca-certificates \
  && git clone https://gitlab.orfeo-toolbox.org/orfeotoolbox/otb.git \
- && cd otb && git checkout $OTB && cd .. \
+ && cd otb && git checkout $OTB \
+# <---------------------------------------- Begin dirty hack
+# This is a dirty hack for release 4.0.0alpha
+# We have to wait that OTB moves from C++14 to C++17
+# See https://gitlab.orfeo-toolbox.org/orfeotoolbox/otb/-/issues/2338
+ && sed -i 's/CMAKE_CXX_STANDARD 14/CMAKE_CXX_STANDARD 17/g' CMakeLists.txt \
+ && echo "" > Modules/Filtering/ImageManipulation/test/CMakeLists.txt \
+ && echo "" > Modules/Segmentation/Conversion/test/CMakeLists.txt \
+ && echo "" > Modules/Radiometry/Indices/test/CMakeLists.txt \
+ && echo "" > Modules/Learning/DempsterShafer/test/CMakeLists.txt \
+ && echo "" > Modules/Feature/Edge/test/CMakeLists.txt \
+ && echo "" > Modules/Core/ImageBase/test/CMakeLists.txt \
+# <---------------------------------------- End dirty hack
+ && cd .. \
  && mkdir -p build \
  && cd build \
  && if $OTBTESTS; then \
@@ -129,8 +151,8 @@ RUN cd /src/otb/build/OTB/build \
       -DOTB_USE_TENSORFLOW=ON -DModule_OTBTensorflow=ON \
       -Dtensorflow_include_dir=/opt/otbtf/include/tf \
       # Forcing TF>=2, this Dockerfile hasn't been tested with v1 + missing link for libtensorflow_framework.so in the wheel
-      -DTENSORFLOW_CC_LIB=/opt/otbtf/lib/libtensorflow_cc.so.2 \
-      -DTENSORFLOW_FRAMEWORK_LIB=/opt/otbtf/lib/python3/site-packages/tensorflow/libtensorflow_framework.so.2 \
+      -DTENSORFLOW_CC_LIB=/opt/otbtf/local/lib/python3.10/dist-packages/tensorflow/libtensorflow_cc.so.2 \
+      -DTENSORFLOW_FRAMEWORK_LIB=/opt/otbtf/local/lib/python3.10/dist-packages/tensorflow/libtensorflow_framework.so.2 \
  && make install -j $(python -c "import os; print(round( os.cpu_count() * $CPU_RATIO ))") \
  # Cleaning
  && ( $GUI || rm -rf /opt/otbtf/bin/otbgui* ) \
@@ -152,10 +174,12 @@ COPY --from=builder /src /src
 # System-wide ENV
 ENV PATH="/opt/otbtf/bin:$PATH"
 ENV LD_LIBRARY_PATH="/opt/otbtf/lib:$LD_LIBRARY_PATH"
-ENV PYTHONPATH="/opt/otbtf/lib/python3/site-packages:/opt/otbtf/lib/python3/dist-packages:/opt/otbtf/lib/otb/python:/src/otbtf"
+ENV PYTHONPATH="/opt/otbtf/lib/python3/dist-packages:/opt/otbtf/lib/otb/python"
 ENV OTB_APPLICATION_PATH="/opt/otbtf/lib/otb/applications"
+RUN pip install -e /src/otbtf
 
-# Default user, directory and command (bash is the entrypoint when using 'docker create')
+# Default user, directory and command (bash is the entrypoint when using
+# 'docker create')
 RUN useradd -s /bin/bash -m otbuser
 WORKDIR /home/otbuser
 
@@ -165,14 +189,18 @@ RUN if $SUDO; then \
       usermod -a -G sudo otbuser \
       && echo "otbuser ALL=(ALL) NOPASSWD:ALL" >> /etc/sudoers; fi
 
-# Set /src/otbtf ownership to otbuser (but you still need 'sudo -i' in order to rebuild TF or OTB)
+# Set /src/otbtf ownership to otbuser (but you still need 'sudo -i' in order
+# to rebuild TF or OTB)
 RUN chown -R otbuser:otbuser /src/otbtf
 
 # This won't prevent ownership problems with volumes if you're not UID 1000
 USER otbuser
+
 # User-only ENV
+ENV PATH="/home/otbuser/.local/bin:$PATH"
 
 # Test python imports
 RUN python -c "import tensorflow"
 RUN python -c "import otbtf, tricks"
 RUN python -c "import otbApplication as otb; otb.Registry.CreateApplication('ImageClassifierFromDeepFeatures')"
+RUN python -c "from osgeo import gdal"