对于目前基于k8s的的spark应用，主要采用两种方式运行
spark原生支持的 spark on k8s
基于k8s的operator的 spark on k8s operator
前者是spark社区支持k8s这种资源管理框架而引入的k8s client的实现
后者是k8s社区为了支持spark而开发的一种operator
code 1 --- bin/spark-submit \ --master k8s://https://localhost:6443 \ --deploy-mode cluster \ --name spark-pi \ --class org.apache.spark.examples.SparkPi \ --conf spark.executor.instances=2 \ --conf "spark.kubernetes.namespace=dev" \ --conf "spark.kubernetes.authenticate.driver.serviceAccountName=monkeyboy" \ --conf "spark.kubernetes.container.image=xxx/dev/spark:cdh-2.6.0-5.13.1" \ --conf "spark.kubernetes.container.image.pullSecrets=monkeyboy" \ --conf "spark.kubernetes.file.upload.path=hdfs:///tmp" \ --conf "spark.kubernetes.container.image.pullPolicy=Always" \ hdfs:///tmp/spark-examples_2.12-3.0.0.jar
code 2 --- apiVersion: "sparkoperator.k8s.io/v1beta2" kind: SparkApplication metadata: name: spark-pi namespace: dev spec: type: Scala mode: cluster image: "gcr.io/spark-operator/spark:v3.0.0" imagePullPolicy: Always mainClass: org.apache.spark.examples.SparkPi mainApplicationFile: "local:///opt/spark/examples/jars/spark-examples_2.12-3.0.0.jar" sparkVersion: "3.0.0" restartPolicy: type: Never volumes: - name: "test-volume" hostPath: path: "/tmp" type: Directory driver: cores: 1 coreLimit: "1200m" memory: "512m" labels: version: 3.0.0 serviceAccount: monkeyboy volumeMounts: - name: "test-volume" mountPath: "/tmp" executor: cores: 1 instances: 1 memory: "512m" labels: version: 3.0.0 volumeMounts: - name: "test-volume" mountPath: "/tmp"
