# A headless service to create DNS records.
apiVersion: v1
kind: Service
metadata:
  name: hdfs-datanode
  labels:
    app: hdfs
    hdfs: datanode
    component: hdfs-datanode
spec:
  ports:
  - port: 9866
    name: fs
  - port: 9864
    name: web
  - port: 9867
    name: ipc
  - port: 8082
    name: metrics
  clusterIP: None
  selector:
    app: hdfs
    hdfs: datanode

---

# A clusterIP service for the web interface.
apiVersion: v1
kind: Service
metadata:
  name: hdfs-datanode-web
  labels:
    app: hdfs
    hdfs: datanode
    component: hdfs-datanode
spec:
  ports:
  - port: 9864
    name: web
  selector:
    app: hdfs
    hdfs: datanode

---

apiVersion: apps/v1
kind: StatefulSet
metadata:
  name: hdfs-datanode
  labels:
    app: hdfs
    hdfs: datanode
spec:
  serviceName: "hdfs-datanode"
  replicas: {{ .Values.hadoop.spec.hdfs.datanode.replicas }}
  updateStrategy:
    type: RollingUpdate
  selector:
    matchLabels:
      app: hdfs
      hdfs: datanode
{{- if .Values.hadoop.spec.hdfs.datanode.labels }}
{{ toYaml .Values.hadoop.spec.hdfs.datanode.labels | indent 6 }}
{{- end }}
  template:
    metadata:
      labels:
        app: hdfs
        hdfs: datanode
{{- if .Values.hadoop.spec.hdfs.datanode.labels }}
{{ toYaml .Values.hadoop.spec.hdfs.datanode.labels | indent 8 }}
{{- end }}
      annotations:
        hadoop-config-hash: {{ include (print $.Template.BasePath "/hadoop/hadoop-config.yaml") . | sha256sum }}
        hadoop-scripts-hash: {{ include (print $.Template.BasePath "/hadoop/hadoop-scripts.yaml") . | sha256sum }}
        hdfs-jmx-config-hash: {{ include (print $.Template.BasePath "/hadoop/hdfs-jmx-config.yaml") . | sha256sum }}
{{- if .Values.hadoop.spec.config.aws.createSecret }}
        hadoop-aws-credentials-hash: {{ include (print $.Template.BasePath "/hadoop/hadoop-aws-credentials.yaml") . | sha256sum }}
{{- end }}
{{- if .Values.hadoop.spec.config.azure.createSecret }}
        hadoop-azure-credentials-hash: {{ include (print $.Template.BasePath "/hadoop/hadoop-azure-credentials.yaml") . | sha256sum }}
{{- end }}
{{- if .Values.hadoop.spec.config.gcs.createSecret }}
        hadoop-gcs-credentials-secret-hash: {{ include (print $.Template.BasePath "/hadoop/hadoop-gcs-credentials-secret.yaml") . | sha256sum }}
{{- end }}
{{- if .Values.hadoop.spec.config.s3Compatible.createSecret }}
        hadoop-s3-compatible-credentials-hash: {{ include (print $.Template.BasePath "/hadoop/hadoop-s3-compatible-credentials.yaml") . | sha256sum }}
{{- end }}
{{- if .Values.hive.spec.config.s3Compatible.ca.createSecret }}
        hive-s3-compatible-ca-secret-hash: {{ include (print $.Template.BasePath "/hive/hive-s3-compatible-ca-secret.yaml") . | sha256sum }}
{{- end }}
{{- if .Values.hadoop.spec.hdfs.datanode.annotations }}
{{ toYaml .Values.hadoop.spec.hdfs.datanode.annotations | indent 8 }}
{{- end }}
    spec:
      terminationGracePeriodSeconds: {{ .Values.hadoop.spec.hdfs.datanode.terminationGracePeriodSeconds }}
{{- if .Values.hadoop.spec.hdfs.securityContext }}
      securityContext:
{{ toYaml .Values.hadoop.spec.hdfs.securityContext | indent 8 }}
{{- end }}
{{- if .Values.hadoop.spec.hdfs.datanode.affinity }}
      affinity:
{{ toYaml .Values.hadoop.spec.hdfs.datanode.affinity | indent 8 }}
{{- end }}
{{- if .Values.hadoop.spec.hdfs.datanode.nodeSelector }}
      nodeSelector:
{{ toYaml .Values.hadoop.spec.hdfs.datanode.nodeSelector | indent 8 }}
{{- end }}
{{- if .Values.hadoop.spec.hdfs.datanode.tolerations }}
      tolerations:
{{ toYaml .Values.hadoop.spec.hdfs.datanode.tolerations | indent 8 }}
{{- end }}
      initContainers:
      # wait-for-namenode exists because for some reason the datanode is unable
      # to connect to the namenode if it starts before the namenode's DNS name
      # is resolvable. We cannot use a clusterIP service for the namenode which
      # would always be resolvable, because on Openshift, clusterIP services
      # NAT loses sourceIPs, breaking HDFS clustering.
      - name: wait-for-namenode
        image: "{{ include "hadoop-image" . }}"
        imagePullPolicy: {{ .Values.hadoop.spec.image.pullPolicy }}
        command:
        - '/bin/bash'
        - '-c'
        - 'until host $NAMENODE_HOST; do echo waiting for $NAMENODE_HOST; sleep 2; done;'
        env:
        - name: NAMENODE_HOST
          value: hdfs-namenode-0.hdfs-namenode
        resources:
          requests:
            memory: "10Mi"
            cpu: "10m"
          limits:
            memory: "50Mi"
            cpu: "50m"
        volumeMounts:
        - name: hdfs-datanode-data
          mountPath: /hadoop/dfs/data
          # we use a subPath to avoid the lost+found directory at the root of
          # the volume effecting the hdfs formating
          subPath: hadoop/dfs/data
        # required for openshift
        - name: namenode-empty
          mountPath: /hadoop/dfs/name
      - name: copy-starter-hadoop
        image: "{{ include "hadoop-image" . }}"
        imagePullPolicy: {{ .Values.hadoop.spec.image.pullPolicy }}
        command: ["/hadoop-scripts/copy-hadoop-config.sh"]
        env:
{{- if or .Values.hadoop.spec.config.azure.secretName .Values.hadoop.spec.config.azure.createSecret }}
        - name: AZURE_STORAGE_ACCOUNT_NAME
          valueFrom:
            secretKeyRef:
              name: "{{ .Values.hadoop.spec.config.azure.secretName | default "hadoop-azure-credentials" }}"
              key: azure-storage-account-name
        - name: AZURE_SECRET_ACCESS_KEY
          valueFrom:
            secretKeyRef:
              name: "{{ .Values.hadoop.spec.config.azure.secretName | default "hadoop-azure-credentials" }}"
              key: azure-secret-access-key
{{- end }}
        volumeMounts:
        - name: hadoop-config
          mountPath: /hadoop-config
        - name: hadoop-starting-config
          mountPath: /hadoop-starting-config
        - name: hadoop-scripts
          mountPath: /hadoop-scripts
{{- if or .Values.hadoop.spec.config.gcs.secretName .Values.hadoop.spec.config.gcs.createSecret }}
        - name: gcs-json
          mountPath: /gcs-json
{{- end }}
      volumes:
      - name: hadoop-scripts
        configMap:
          name: hadoop-scripts
          defaultMode: 0775
      containers:
      - name: hdfs-datanode
        image: "{{ include "hadoop-image" . }}"
        imagePullPolicy: {{ .Values.hadoop.spec.image.pullPolicy }}
        command: ["/hadoop-scripts/entrypoint.sh"]
        args: ["/hadoop-scripts/datanode-entrypoint.sh"]
        env:
        - name: HADOOP_LOGLEVEL
          value: "{{ upper .Values.hadoop.spec.hdfs.config.logLevel }}"
{{- if or .Values.hadoop.spec.config.aws.secretName .Values.hadoop.spec.config.aws.createSecret }}
        - name: AWS_ACCESS_KEY_ID
          valueFrom:
            secretKeyRef:
              name: "{{ .Values.hadoop.spec.config.aws.secretName | default "hadoop-aws-credentials" }}"
              key: aws-access-key-id
        - name: AWS_SECRET_ACCESS_KEY
          valueFrom:
            secretKeyRef:
              name: "{{ .Values.hadoop.spec.config.aws.secretName | default "hadoop-aws-credentials" }}"
              key: aws-secret-access-key
{{- end }}
{{- if .Values.hadoop.spec.config.s3Compatible.endpoint }}
        - name: AWS_ACCESS_KEY_ID
          valueFrom:
            secretKeyRef:
              name: "{{ .Values.hadoop.spec.config.s3Compatible.secretName | default "hadoop-s3-compatible-credentials" }}"
              key: aws-access-key-id
        - name: AWS_SECRET_ACCESS_KEY
          valueFrom:
            secretKeyRef:
              name: "{{ .Values.hadoop.spec.config.s3Compatible.secretName | default "hadoop-s3-compatible-credentials" }}"
              key: aws-secret-access-key
{{- end }}
        - name: MY_NODE_NAME
          valueFrom:
            fieldRef:
              fieldPath: spec.nodeName
        - name: MY_POD_NAME
          valueFrom:
            fieldRef:
              fieldPath: metadata.name
        - name: MY_POD_NAMESPACE
          valueFrom:
            fieldRef:
              fieldPath: metadata.namespace
        - name: MY_MEM_REQUEST
          valueFrom:
            resourceFieldRef:
              containerName: hdfs-datanode
              resource: requests.memory
        - name: MY_MEM_LIMIT
          valueFrom:
            resourceFieldRef:
              containerName: hdfs-datanode
              resource: limits.memory
{{- if .Values.hadoop.spec.hdfs.datanode.config.jvm.initialRAMPercentage }}
        - name: JVM_INITIAL_RAM_PERCENTAGE
          value: "{{ .Values.hadoop.spec.hdfs.datanode.config.jvm.initialRAMPercentage }}"
{{- end }}
{{- if .Values.hadoop.spec.hdfs.datanode.config.jvm.maxRAMPercentage }}
        - name: JVM_MAX_RAM_PERCENTAGE
          value: "{{ .Values.hadoop.spec.hdfs.datanode.config.jvm.maxRAMPercentage }}"
{{- end }}
{{- if .Values.hadoop.spec.hdfs.datanode.config.jvm.maxRAMPercentage }}
        - name: JVM_MIN_RAM_PERCENTAGE
          value: "{{ .Values.hadoop.spec.hdfs.datanode.config.jvm.minRAMPercentage }}"
{{- end }}
        ports:
        - containerPort: 9864
          name: http
        - containerPort: 9866
          name: fs
        - containerPort: 9867
          name: ipc
        - containerPort: 8082
          name: metrics
        volumeMounts:
        - name: hadoop-config
          mountPath: /hadoop-config
        - name: hadoop-starting-config
          mountPath: /hadoop-starting-config
        - name: hadoop-scripts
          mountPath: /hadoop-scripts
        - name: hdfs-jmx-config
          mountPath: /opt/jmx_exporter/config
        - name: hdfs-service-ca-bundle
          mountPath: /var/run/configmaps/service-ca-bundle
          readOnly: true
{{- if or .Values.hadoop.spec.config.s3Compatible.ca.secretName .Values.hadoop.spec.config.s3Compatible.ca.createSecret }}
        - name: s3-compatible-ca
          mountPath: /s3-compatible-ca
{{- end }}
        - name: hdfs-datanode-data
          mountPath: /hadoop/dfs/data
          # we use a subPath to avoid the lost+found directory at the root of
          # the volume effecting the hdfs formating
          subPath: hadoop/dfs/data
        # required for openshift
        - name: namenode-empty
          mountPath: /hadoop/dfs/name
        - name: hadoop-logs
          mountPath: /opt/hadoop/logs
        livenessProbe:
          exec:
            command: ["/hadoop-scripts/check-datanode-healthy.sh" ]
          initialDelaySeconds: 90
          periodSeconds: 30
          failureThreshold: 2
          successThreshold: 1
        resources:
{{ toYaml .Values.hadoop.spec.hdfs.datanode.resources | indent 10 }}
      serviceAccount: hdfs
{{- if .Values.hadoop.spec.image.pullSecrets }}
      imagePullSecrets:
{{ toYaml .Values.hadoop.spec.image.pullSecrets | indent 8 }}
{{- end }}
      volumes:
      - name: hadoop-config
        emptyDir: {}
      - name: hadoop-starting-config
        secret:
          secretName: "{{ .Values.hadoop.spec.configSecretName }}"
          defaultMode: 0775
      - name: hadoop-scripts
        configMap:
          name: hadoop-scripts
          defaultMode: 0775
      - name: hdfs-jmx-config
        configMap:
          name: hdfs-jmx-config
      - name: hdfs-service-ca-bundle
        configMap:
          name: service-ca-bundle
          optional: true
      - name: namenode-empty
        emptyDir: {}
      - name: hadoop-logs
        emptyDir: {}
{{- if or .Values.hadoop.spec.config.gcs.secretName .Values.hadoop.spec.config.gcs.createSecret }}
      - name: gcs-json
        secret:
          secretName: "{{ .Values.hadoop.spec.config.gcs.secretName | default "hadoop-gcs-credentials" }}"
{{- end }}
{{- if or .Values.hadoop.spec.config.s3Compatible.ca.secretName .Values.hadoop.spec.config.s3Compatible.ca.createSecret }}
      - name: s3-compatible-ca
        secret:
          secretName: {{ .Values.hadoop.spec.config.s3Compatible.ca.secretName | default "hadoop-s3-compatible-ca" }}
{{- end }}
  volumeClaimTemplates:
  - metadata:
      name: "hdfs-datanode-data"
      labels:
        app: hdfs
        hdfs: datanode
    spec:
      accessModes: ["ReadWriteOnce"]
      storageClassName: {{ .Values.hadoop.spec.hdfs.datanode.storage.class }}
      resources:
        requests:
          storage: {{ .Values.hadoop.spec.hdfs.datanode.storage.size }}
