diff --git a/kcdmx2024/README.md b/kcdmx2024/README.md new file mode 100644 index 0000000..02f1ab2 --- /dev/null +++ b/kcdmx2024/README.md @@ -0,0 +1,89 @@ +# KCD Mexico workshop: building a cloud-native platform for ML training from scratch + +This is the logical architecture you'll be deploying and using: + +![](./img/logicalarch.png) + +## Pre-requisites + +1. Docker Desktop +2. kubectl +3. minikube +4. python 3.9+ + +## Preparing a Kubernetes environment + +1. Launch a new cluster + +```bash +minikube start --driver=docker +``` + +2. Download the manifest for the Flyte dependencies: +```bash +curl -sl https://raw.githubusercontent.com/flyte-conference-talks/kcdmexico-2024/manifests/flyte-resources.yaml > flyte-resources.yaml +``` +3. Submit the manifest: +```bash +kubectl create -f flyte-resources.yaml +``` +3. Install Flyte: +```bash +helm install flyte-binary flyteorg/flyte-binary --values values.yaml -n flyte +``` +4. In three separate Terminal windows, start port-forwarding sessions to the following components: + +Web console +``` +kubectl -n flyte port-forward service/flyte-binary-http 8088:8088 +``` + +API endpoint +``` +kubectl -n flyte port-forward service/flyte-binary-grpc 8089:8089 +``` +minio (blob storage) +``` +kubectl -n flyte port-forward service/minio 9000:9000 +``` + +5. Edit the `$HOME/.flyte/config.yaml` file to reflect the following: +```yaml +admin: + endpoint: localhost:8089 + insecure: true + authType: Pkce +``` +6. Add and entry to your local DNS file so your `pyflyte` client is able to resolve the `minio` service name: +```bash +sudo vi /etc hosts + +# Host Database +# +# localhost is used to configure the loopback interface +# when the system is booting. Do not change this entry. +## +127.0.0.1 minio.flyte.svc.cluster.local +``` + +7. Download this demo workflow or simply start developing your own: + +``` bash +curl -sl https://raw.githubusercontent.com/davidmirror-ops/flyte-the-hard-way/main/docs/on-premises/microk8s/demo.py > demo.py +``` + +8. Submit the workflow: +``` bash +pyflyte run --remote demo.py wf +``` +Example output: +``` bash +Go to https://localhost:8089/console/projects/flytesnacks/domains/development/executions/f63a3e948256f4fd1b81 to see execution in the console. +``` +> NOTE: instead of `8089` use `8088` as the port you connect to in the browser with the above link. All these limitations don't exist in production environments that make use of an Ingress or Service Mesh. + +Follow the link and observe your workflow succeeding! +> The first run will take some time due to the download of the flytekit docker image + +### Congratulations +You have setup a working Flyte environment on minikube diff --git a/kcdmx2024/img/logicalarch.png b/kcdmx2024/img/logicalarch.png new file mode 100644 index 0000000..66d6f70 Binary files /dev/null and b/kcdmx2024/img/logicalarch.png differ diff --git a/kcdmx2024/manifests/flyte-resources.yaml b/kcdmx2024/manifests/flyte-resources.yaml new file mode 100644 index 0000000..1249a7e --- /dev/null +++ b/kcdmx2024/manifests/flyte-resources.yaml @@ -0,0 +1,178 @@ +apiVersion: v1 +kind: Namespace +metadata: + name: flyte + labels: + name: flyte +--- +apiVersion: v1 +kind: Secret +metadata: + name: flyte-binary-inline-config-secret + namespace: flyte +type: Opaque +stringData: + 202-database-secrets.yaml: | + database: + postgres: + password: "postgres" +--- +apiVersion: v1 +kind: PersistentVolumeClaim +metadata: + name: postgresql-pvc + namespace: flyte +spec: + accessModes: + - ReadWriteOnce + resources: + requests: + storage: 10Gi +--- +apiVersion: v1 +kind: PersistentVolumeClaim +metadata: + name: minio-pvc + namespace: flyte +spec: + accessModes: + - ReadWriteOnce + resources: + requests: + storage: 20Gi +--- +apiVersion: v1 +kind: Service +metadata: + name: postgres + namespace: flyte + labels: + app.kubernetes.io/name: postgres +spec: + type: NodePort + ports: + - name: postgres + port: 5432 + nodePort: 30089 + protocol: TCP + targetPort: postgres + selector: + app.kubernetes.io/name: postgres +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: postgres + namespace: flyte + labels: + app.kubernetes.io/name: postgres +spec: + replicas: 1 + selector: + matchLabels: + app.kubernetes.io/name: postgres + template: + metadata: + labels: + app.kubernetes.io/name: postgres + spec: + containers: + - image: "ecr.flyte.org/ubuntu/postgres:13-21.04_beta" + imagePullPolicy: "IfNotPresent" + name: postgres + env: + - name: POSTGRES_PASSWORD + value: postgres #set your own + - name: POSTGRES_USER + value: postgres + - name: POSTGRES_DB + value: flyte + ports: + - containerPort: 5432 + name: postgres + resources: + limits: + cpu: 1000m + memory: 512Mi + requests: + cpu: 10m + memory: 128Mi + volumeMounts: + - name: postgres-storage + mountPath: /var/lib/postgresql/data + volumes: + - name: postgres-storage + persistentVolumeClaim: + claimName: postgresql-pvc +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: minio + namespace: flyte + labels: + app.kubernetes.io/name: minio +spec: + replicas: 1 + selector: + matchLabels: + app.kubernetes.io/name: minio + template: + metadata: + labels: + app.kubernetes.io/name: minio + spec: + containers: + - image: "bitnami/minio:2023.7.18-debian-11-r2" + imagePullPolicy: "IfNotPresent" + name: minio + env: + #change the following values if needed. Make sure to apply these changes also to the corresponding fields in the local-values.yaml file + - name: MINIO_ROOT_USER + value: minio + - name: MINIO_ROOT_PASSWORD + value: miniostorage + - name: MINIO_DEFAULT_BUCKETS + value: flyte + ports: + - containerPort: 9000 + name: minio + - containerPort: 9001 + name: minio-console + resources: + limits: + cpu: 200m + memory: 512Mi + requests: + cpu: 10m + memory: 128Mi + volumeMounts: + - name: minio-storage + mountPath: /data + volumes: + - name: minio-storage + persistentVolumeClaim: + claimName: minio-pvc +--- +apiVersion: v1 +kind: Service +metadata: + name: minio + namespace: flyte + labels: + app.kubernetes.io/name: minio +spec: + type: NodePort + ports: + - name: minio + nodePort: 30084 + port: 9000 + protocol: TCP + targetPort: minio + - name: minio-console + nodePort: 30088 + port: 9001 + protocol: TCP + targetPort: minio-console + selector: + app.kubernetes.io/name: minio \ No newline at end of file diff --git a/kcdmx2024/values.yaml b/kcdmx2024/values.yaml new file mode 100644 index 0000000..4dbe078 --- /dev/null +++ b/kcdmx2024/values.yaml @@ -0,0 +1,39 @@ +configuration: + database: + username: postgres + host: postgres.flyte.svc.cluster.local + dbname: flyte + storage: + type: minio + metadataContainer: flyte #This is the default bucket created with Minio. Controlled by the MINIO_DEFAULT_BUCKETS env var in the local-flyte-resources.yaml manifest + userDataContainer: flyte + provider: s3 + providerConfig: + s3: + authType: "accesskey" + endpoint: "http://minio.flyte.svc.cluster.local:57149" + accessKey: "minio" + secretKey: "miniostorage" #If you need to change this parameters, refer to the local-flyte-resources.yaml manifest and adjust the MINIO_ROOT_PASSWORD env var + disableSSL: "true" + secure: "false" + v2Signing: "true" + + inlineSecretRef: flyte-binary-inline-config-secret + inline: + plugins: + k8s: + inject-finalizer: true + default-env-vars: + - FLYTE_AWS_ENDPOINT: "http://minio.flyte.svc.cluster.local:9000" + - FLYTE_AWS_ACCESS_KEY_ID: "minio" + - FLYTE_AWS_SECRET_ACCESS_KEY: "miniostorage" #Use the same value as the MINIO_ROOT_PASSWORD + + task_resources: + defaults: + cpu: 100m + memory: 500Mi #change default requested resources and limits to fit your needs + limits: + memory: 2Gi + +serviceAccount: + create: true