Versions Compared

Key

  • This line was added.
  • This line was removed.
  • Formatting was changed.
Comment: Add force nfs-provisioner runs on NFS server node

...

We are using Kubernetes replicas to achieve the SDNC cluster deployment (see details from 4. Deploy/Undeploy the About SDN-C ClusterClustering for the desired goal).

This only needs to be done one time and, at the moment, all modifications are done manually (they can be automated via scripting in the future when the needs come up).

...

The following is the list of SDNC deployment templates which need to be modified for an SDN-C cluster deployment:

...

.

under directory {$OOM}/kubernetes/sdnc
#Files underChanged/Added fields and values
1values.yamlDefined a new variable "mysql: mysql:5.6"

...

under directory 
{$OOM}/kubernetes/sdnc/templates
#file nameChanged/Added fields and values
1
db-deployment.yaml
FieldNew valueOld value
.apiVersion

apps/v1beta1 1

extensions/v1beta1

.kindStatefulSetDeployment
.spec.serviceName"dbhost" 2N/A
.spec.replicas2N/A
.spec.initContainers
Code Block
titleInit Containers
collapsetrue
initContainers:
      - name: init-mysql
        image: {{ .Values.image.mysql }}
        imagePullPolicy: {{ .Values.pullPolicy }}
        command:
        - bash
        - "-c"
        - |
          set -ex
          # Generate mysql server-id from pod ordinal index.
          [[ `hostname` =~ -([0-9]+)$ ]] || exit 1
          ordinal=${BASH_REMATCH[1]}
          echo [mysqld] > /mnt/conf.d/server-id.cnf
          # Add an offset to avoid reserved server-id=0 value.
          echo server-id=$((100 + $ordinal)) >> /mnt/conf.d/server-id.cnf
          # Copy appropriate conf.d files from config-map to emptyDir.
          if [[ $ordinal -eq 0 ]]; then
            cp /mnt/config-map/master.cnf /mnt/conf.d/
          else
            cp /mnt/config-map/slave.cnf /mnt/conf.d/
          fi
        volumeMounts:
        - name: conf
          mountPath: /mnt/conf.d
        - name: config-map
          mountPath: /mnt/config-map
      - name: clone-mysql
        image: gcr.io/google-samples/xtrabackup:1.0
        command:
        - bash
        - "-c"
        - |
          set -ex
          # Skip the clone if data already exists.
          [[ -d /var/lib/mysql/mysql ]] && exit 0
          # Skip the clone on master (ordinal index 0).
          [[ `hostname` =~ -([0-9]+)$ ]] || exit 1
          ordinal=${BASH_REMATCH[1]}
          [[ $ordinal -eq 0 ]] && exit 0
          # Clone data from previous peer.
          ncat --recv-only sdnc-dbhost-$(($ordinal-1)).dbhost.{{ .Values.nsPrefix }}-sdnc 3307 | xbstream -x -C /var/lib/mysql
          # Prepare the backup.
          xtrabackup --prepare --target-dir=/var/lib/mysql
        volumeMounts:
        - name: sdnc-data
          mountPath: /var/lib/mysql
          subPath: mysql
        - name: conf
          mountPath: /etc/mysql/conf.d


N/A
.spec.containers
Code Block
titleContainers
collapsetrue
containers:
      - env:
        - name: MYSQL_ROOT_PASSWORD
          value: openECOMP1.0
        - name: MYSQL_ROOT_HOST
          value: '%'
        image: {{ .Values.image.mysqlServer }}
        imagePullPolicy: {{ .Values.pullPolicy }}
        name: sdnc-db-container
        volumeMounts:
        - mountPath: /var/lib/mysql
          name: sdnc-data
          subPath: mysql
        - name: conf
          mountPath: /etc/mysql/conf.d
        ports:
        - containerPort: 3306
        resources:
          requests:
            cpu: 500m
            memory: 1Gi
        livenessProbe:
          exec:
            command: ["mysqladmin", "ping"]
          initialDelaySeconds: 30
          periodSeconds: 10
          timeoutSeconds: 5
        readinessProbe:
          tcpSocket:
            port: 3306
          initialDelaySeconds: 5
          periodSeconds: 10
      - name: xtrabackup
        image: gcr.io/google-samples/xtrabackup:1.0
        env:
        - name: MYSQL_ROOT_PASSWORD
          value: openECOMP1.0
        - name: MYSQL_ROOT_HOST
          value: '%'
        ports:
        - name: xtrabackup
          containerPort: 3307
        command:
        - bash
        - "-c"
        - |
          set -ex
          cd /var/lib/mysql

          # Determine binlog position of cloned data, if any.
          if [[ -f xtrabackup_slave_info ]]; then
            # XtraBackup already generated a partial "CHANGE MASTER TO" query
            # because we're cloning from an existing slave.
            mv xtrabackup_slave_info change_master_to.sql.in
            # Ignore xtrabackup_binlog_info in this case (it's useless).
            rm -f xtrabackup_binlog_info
          elif [[ -f xtrabackup_binlog_info ]]; then
            # We're cloning directly from master. Parse binlog position.
            [[ `cat xtrabackup_binlog_info` =~ ^(.*?)[[:space:]]+(.*?)$ ]] || exit 1
            rm xtrabackup_binlog_info
            echo "CHANGE MASTER TO MASTER_LOG_FILE='${BASH_REMATCH[1]}',\
                  MASTER_LOG_POS=${BASH_REMATCH[2]}" > change_master_to.sql.in
          fi

          # Check if we need to complete a clone by starting replication.
          if [[ -f change_master_to.sql.in ]]; then
            echo "Waiting for mysqld to be ready (accepting connections)"
            until mysql --user=root --password=$MYSQL_ROOT_PASSWORD -h 127.0.0.1 -e "SELECT 1"; do sleep 1; done

            echo "Initializing replication from clone position"
            # In case of container restart, attempt this at-most-once.
            mv change_master_to.sql.in change_master_to.sql.orig
            mysql --user=root --password=$MYSQL_ROOT_PASSWORD -h 127.0.0.1 <<EOF
          $(<change_master_to.sql.orig),
            MASTER_HOST='sdnc-dbhost-0.dbhost.{{ .Values.nsPrefix }}-sdnc',
            MASTER_USER="root",
            MASTER_PASSWORD=$MYSQL_ROOT_PASSWORD,
            MASTER_CONNECT_RETRY=10;
          START SLAVE;
          EOF
          fi

          # Start a server to send backups when requested by peers.
          exec ncat --listen --keep-open --send-only --max-conns=1 3307 -c \
            "xtrabackup --backup --slave-info --stream=xbstream --host=127.0.0.1 --user=root --password=$MYSQL_ROOT_PASSWORD"
        volumeMounts:
        - name: sdnc-data
          mountPath: /var/lib/mysql
          subPath: mysql
        - name: conf
          mountPath: /etc/mysql/conf.d
        resources:
          requests:
            cpu: 100m
            memory: 100Mi
Code Block
titleContainers
collapsetrue
containers:
      - env:
        - name: MYSQL_ROOT_PASSWORD
          value: openECOMP1.0
        - name: MYSQL_ROOT_HOST
          value: '%'
        image: {{ .Values.image.mysqlServer }}
        imagePullPolicy: {{ .Values.pullPolicy }}
        name: sdnc-db-container
        volumeMounts:
        - mountPath: /etc/localtime
          name: localtime
          readOnly: true
        - mountPath: /var/lib/mysql
          name: sdnc-data
        ports:
        - containerPort: 3306
        readinessProbe:
          tcpSocket:
            port: 3306
          initialDelaySeconds: 5
          periodSeconds: 10
.spec.volumes
Code Block
titleDynamic Volume Claim
collapsetrue
volumeClaimTemplates:
  - metadata:
     name: sdnc-data
     annotations:
       volume.beta.kubernetes.io/storage-class: "{{ .Values.nsPrefix }}-sdnc-data"
    spec:
      accessModes: ["ReadWriteMany"]
      resources:
        requests:
          storage: 1Gi
Code Block
titleStatic Volume Claim
collapsetrue
volumes:
      - name: localtime
        hostPath:
          path: /etc/localtime
      - name: sdnc-data
        persistentVolumeClaim:
          claimName: sdnc-db
2
sdnc-deployment.yaml



FieldNew valueOld value
.apiVersion

apps/v1beta1 1

extensions/v1beta1

.kindStatefulSetDeployment
.spec.serviceName"sdnhostcluster" 2N/A
.spec.replicas3N/A
.spec.podManagementPolicy"Parallel" 3N/A
.spec.containers.ports  4

- containerPort: 2550
- containerPort: 8080

N/A

.spec.containers.volumeMounts 5.a 

- mountPath: /opt/onap/sdnc/bin/startODL.sh
name: sdnc-startodl
- mountPath: /opt/opendaylight/current/deploy
name: sdnc-deploy

N/A
 .spec.volumes 5.b 

- name: sdnc-deploy
hostPath:
path: /home/ubuntu/cluster/deploy
- name: sdnc-startodl
hostPath:
path: /home/ubuntu/cluster/script/cluster-startODL.sh

N/A
3
web-deployment.yaml
FieldNew valueOld value
.apiVersion

apps/v1beta1 1

extensions/v1beta1

.kindStatefulSetDeployment
.spec.serviceName"sdnc-portal" 2N/A
.spec.replicas3N/A
.spec.podManagementPolicy"Parellel" 3N/A
4sdnc-pv-pvc.yaml

Changed from Static volume mounts (PV and PVCs) to Dynamic volume mounts

New ValueOld Value
Code Block
titleDynamic Volume Provisioner
collapsetrue
kind: Service
apiVersion: v1
metadata:
  name: nfs-provisioner
  namespace: "{{ .Values.nsPrefix }}-sdnc"
  labels:
    app: nfs-provisioner
spec:
  ports:
    - name: nfs
      port: 2049
    - name: mountd
      port: 20048
    - name: rpcbind
      port: 111
    - name: rpcbind-udp
      port: 111
      protocol: UDP
  selector:
    app: nfs-provisioner
---
kind: Deployment
apiVersion: extensions/v1beta1
metadata:
  name: nfs-provisioner
  namespace: "{{ .Values.nsPrefix }}-sdnc"
spec:
  replicas: 1
  strategy:
    type: Recreate 
  template:
    metadata:
      labels:
        app: nfs-provisioner
    spec:
      containers:
        - name: nfs-provisioner
          image: quay.io/kubernetes_incubator/nfs-provisioner:v1.0.8
          ports:
            - name: nfs
              containerPort: 2049
            - name: mountd
              containerPort: 20048
            - name: rpcbind
              containerPort: 111
            - name: rpcbind-udp
              containerPort: 111
              protocol: UDP
          securityContext:
            capabilities:
              add:
                - DAC_READ_SEARCH
                - SYS_RESOURCE
          args:
            - "-provisioner=sdnc/nfs"
          env:
            - name: POD_IP
              valueFrom:
                fieldRef:
                  fieldPath: status.podIP
            - name: SERVICE_NAME
              value: nfs-provisioner
            - name: POD_NAMESPACE
              valueFrom:
                fieldRef:
                  fieldPath: metadata.namespace
          imagePullPolicy: "IfNotPresent"
          volumeMounts:
            - name: export-volume
              mountPath: /export
      volumes:
        - name: export-volume
          hostPath:
            path: /dockerdata-nfs/{{ .Values.nsPrefix }}/sdnc/data
---
kind: StorageClass
apiVersion: storage.k8s.io/v1
metadata:
  name: "{{ .Values.nsPrefix }}-sdnc-data"
  namespace: "{{ .Values.nsPrefix }}-sdnc"
provisioner: sdnc/nfs
Code Block
titleStatic volume
collapsetrue
kind: PersistentVolume
metadata:
  name: "{{ .Values.nsPrefix }}-sdnc-db"
  namespace: "{{ .Values.nsPrefix }}-sdnc"
  labels:
    name: "{{ .Values.nsPrefix }}-sdnc-db"
spec:
  capacity:
    storage: 2Gi
  accessModes:
    - ReadWriteMany
  persistentVolumeReclaimPolicy: Retain
  hostPath:
    path: /dockerdata-nfs/{{ .Values.nsPrefix }}/sdnc/data
---
kind: PersistentVolumeClaim
apiVersion: v1
metadata:
  name: sdnc-db
  namespace: "{{ .Values.nsPrefix }}-sdnc"
spec:
  accessModes:
    - ReadWriteMany
  resources:
    requests:
      storage: 2Gi
  selector:
    matchLabels:
      name: "{{ .Values.nsPrefix }}-sdnc-db"
5
all-services.yaml



  • Add a headless service of sdnc 6
Code Block
---
apiVersion: v1
kind: Service
metadata:
 name: sdnhostcluster
 namespace: onap-sdnc
 labels:
   app: sdnc
 annotations:
   service.alpha.kubernetes.io/tolerate-unready-endpoints: "true"
spec:
 ports:
 - name: "sdnc-cluster-port"
   port: 2550
 clusterIP: None
 selector:
   app: sdnc
 sessionAffinity: None
 type: ClusterIP

(Find sdnhost service and add expose one more port under .spec.ports)

Code Block
- name: "sdnc-jolokia-port-8080"
  port: 9090
  targetPort: 8080
  nodePort: {{ .Values.nodePortPrefix }}00
#Files under {$OOM}/kubernetes/sdncChanged/Added fields and values1values.yamlDefined a new variable "mysql: mysql:5.6"


Notes
:

  1. Use .apiVersion "apps/v1beta1" for the Kubernetes version before 1.8.0; otherwise, use .apiVersion "apps/v1beta2"
    • Check the Kubernetes version using the command "kubectl version"

      Expand
      titleExample of kubernetes version 1.7.7

      Client Version: version.Info{Major:"1", Minor:"8", GitVersion:"v1.8.2", GitCommit:"bdaeafa71f6c7c04636251031f93464384d54963", GitTreeState:"clean", BuildDate:"2017-10-24T19:48:57Z", GoVersion:"go1.8.3", Compiler:"gc", Platform:"linux/amd64"}
      Server Version: version.Info{Major:"1", Minor:"7+", GitVersion:"v1.7.7-rancher1", GitCommit:"a1ea37c6f6d21f315a07631b17b9537881e1986a", GitTreeState:"clean", BuildDate:"2017-10-02T21:33:08Z", GoVersion:"go1.8.3", Compiler:"gc", Platform:"linux/amd64"}

      Expand
      titleExample of kubernetes version 1.8.3

      Client Version: version.Info{Major:"1", Minor:"8", GitVersion:"v1.8.3", GitCommit:"f0efb3cb883751c5ffdbe6d515f3cb4fbe7b7acd", GitTreeState:"clean", BuildDate:"2017-11-08T18:39:33Z", GoVersion:"go1.8.3", Compiler:"gc", Platform:"linux/amd64"}

      Server Version: version.Info{Major:"1", Minor:"8+", GitVersion:"v1.8.3-rancher1", GitCommit:"beb8311a9f114ba92558d8d771a81b7fb38422ae", GitTreeState:"clean", BuildDate:"2017-11-14T00:54:19Z", GoVersion:"go1.8.3", Compiler:"gc", Platform:"linux/amd64"}

  2. The value must align with the associated service name in the all_services.yaml file under the same directory.
  3. By default, .spec.podManagementPolicy has the value "OrderReady".
    • With the value "OrderReady", the Kubernetes pod management tells the StatefulSet controller to respect the ordering guarantees, waiting for a Pod to become Running and Ready or completely terminate and then launching or terminating another Pod.
    • With the value "Parallel", the Kubernetes pod management tells the StatefulSet controller to launch or terminate all Pods in parallel, not waiting for Pods to become Running and Ready or completely terminated prior to launching or terminating another Pod.
  4. Export 2 new ports for
  5. Since StartODL.sh has to be changed to enable clusters to function, two paths must be mounted:
    1. mount /home/ubuntu/cluster/script/cluster-startODL.sh (local) to replace /opt/onap/sdnc/bin/startODL.sh (docker), so that we can use our local updated script with cluster config.
    2. mount /home/ubuntu/cluster/deploy (local) to /opt/opendaylight/current/deploy (docker), so that we can dynamically deploy test bundles outside pods.
  6. The newly added headless service is SDNC pods in SDNC cluster to be able to find each other with fixed FQDN directly.

Make nfs-provisioner Pod Runs On Node Where NFS Server Runs

On the node where you have configured nfs server, run the following:

#PurposeCommand and Example
1find the node name
Expand
titlefind nfs server node

Run command "ps -ef|grep nfs", you should

  • node with nfs server runs nfsd process:

ubuntu@sdnc-k8s:~$ ps -ef|grep nfs
root 3473 2 0 Dec07 ? 00:00:00 [nfsiod]
root 11072 2 0 Dec06 ? 00:00:00 [nfsd4_callbacks]
root 11074 2 0 Dec06 ? 00:00:00 [nfsd]
root 11075 2 0 Dec06 ? 00:00:00 [nfsd]
root 11076 2 0 Dec06 ? 00:00:00 [nfsd]
root 11077 2 0 Dec06 ? 00:00:00 [nfsd]
root 11078 2 0 Dec06 ? 00:00:00 [nfsd]
root 11079 2 0 Dec06 ? 00:00:03 [nfsd]
root 11080 2 0 Dec06 ? 00:00:13 [nfsd]
root 11081 2 0 Dec06 ? 00:00:42 [nfsd]
ubuntu@sdnc-k8s:~$

  • node with nfs client runs nfs svc process:

ubuntu@sdnc-k8s-2:~$ ps -ef|grep nfs
ubuntu 5911 5890 0 20:10 pts/0 00:00:00 grep --color=auto nfs
root 18739 2 0 Dec06 ? 00:00:00 [nfsiod]
root 18749 2 0 Dec06 ? 00:00:00 [nfsv4.0-svc]
ubuntu@sdnc-k8s-2:~$

kubectl get node

Expand
titleExample of response

ubuntu@sdnc-k8s:~$ kubectl get node
NAME STATUS ROLES AGE VERSION
sdnc-k8s Ready master 6d v1.8.4
sdnc-k8s-2 Ready <none> 6d v1.8.4
ubuntu@sdnc-k8s:~$

2set label on the node

kubectl label nodes <NODE_NAME_FROM_LAST_STEP> disktype=ssd

Expand
titleAn example

ubuntu@sdnc-k8s:~$ kubectl label nodes sdnc-k8s disktype=ssd

node "sdnc-k8s" labeled

3check the label has been set on the node

kubectl get node --show-labels

Expand
titleAn example

ubuntu@sdnc-k8s:~$ kubectl get node --show-labels
NAME STATUS ROLES AGE VERSION LABELS
sdnc-k8s Ready master 6d v1.8.4 beta.kubernetes.io/arch=amd64,beta.kubernetes.io/os=linux,disktype=ssd,kubernetes.io/hostname=sdnc-k8s,node-role.kubernetes.io/master=
sdnc-k8s-2 Ready <none> 6d v1.8.4 beta.kubernetes.io/arch=amd64,beta.kubernetes.io/os=linux,kubernetes.io/hostname=sdnc-k8s-2
ubuntu@sdnc-k8s:~$

4update nfs-provisioner pod template to force it running on the nfs server node

In sdnc-pv-pv.yaml file, add “spec.template.spec.nodeSelector” for pod “nfs-provisioner

Expand
titleAn example of the nfs-provisioner pod with nodeSelector

Image Added


Create New Script: cluster-startODL.sh

...