ssh to the host e.g. ssh <your id>@172.15.3.28
Switch to root user: sudo su
Change to home directory for root: cd or cd ~
Change to oom/kubernetes directory: cd oom/kubernetes
Make the changes in the policy helm charts.
Run make command: make policy
Then make onap: make SKIP_LINT=TRUE onap
Update repo: helm repo update
To list deployment names: helm list
Undeploy the existing policy deployment : helm undeploy <deployment name>
Redeploy: helm deploy <deployment-name> local/onap -f ~/integration-override.yaml --namespace onap --verbose
Further information can be found here: /oom/docs/oom_quickstart_guide.rst
POLICY-3487 - Getting issue details... STATUS
Update Steps for POLICY-3487
copy db_migrator_policy_init.sh to kubernetes/policy/resources/config/
#!/bin/sh
/opt/app/policy/bin/prepare_upgrade.sh ${SQL_DB}
/opt/app/policy/bin/db-migrator -s ${SQL_DB} -o upgrade
/opt/app/policy/bin/db-migrator -s ${SQL_DB} -o report
Update kubernetes/policy/templates/configmap.yaml (configmap.yaml)
apiVersion: v1
kind: ConfigMap
metadata:
name: {{ include "common.fullname" . }}-db-configmap
namespace: {{ include "common.namespace" . }}
labels:
app: {{ include "common.name" . }}
chart: {{ .Chart.Name }}-{{ .Chart.Version | replace "+" "_" }}
release: {{ include "common.release" . }}
heritage: {{ .Release.Service }}
data:
{{ tpl (.Files.Glob "resources/config/*.sh").AsConfig . | indent 2 }}
Update kubernetes/policy/values.yaml (values.yaml)
################################################################# # DB configuration defaults. ################################################################# repository: nexus3.onap.org:10001 pullPolicy: Always mariadb: image: mariadb:10.5.8 dbmigrator: image: onap/policy-db-migrator:2.3.0-SNAPSHOT schema: policyadmin policy_home: "/opt/app/policy" subChartsOnly: enabled: true
Update kubernetes/policy/templates/job.yaml (job.yaml)
apiVersion: batch/v1
kind: Job
metadata:
name: {{ include "common.release" . }}-policy-galera-config
namespace: {{ include "common.namespace" . }}
labels:
app: {{ include "common.name" . }}-job
release: {{ include "common.release" . }}
spec:
template:
metadata:
labels:
app: {{ include "common.name" . }}-job
release: {{ include "common.release" . }}
spec:
imagePullSecrets:
- name: "{{ include "common.namespace" . }}-docker-registry-key"
initContainers:
#This container checks that all galera instances are up before initializing it.
- name: {{ include "common.name" . }}-readiness
image: {{ include "repositoryGenerator.image.readiness" . }}
imagePullPolicy: {{ .Values.global.pullPolicy | default .Values.pullPolicy }}
command:
- /app/ready.py
- --container-name
- {{ index .Values "mariadb-galera" "service" "name" }}
env:
- name: NAMESPACE
valueFrom:
fieldRef:
apiVersion: v1
fieldPath: metadata.namespace
- name: {{ include "common.release" . }}-policy-galera-config
image: {{ include "repositoryGenerator.dockerHubRepository" . }}/{{ .Values.mariadb.image }}
imagePullPolicy: {{ .Values.global.pullPolicy | default .Values.pullPolicy }}
volumeMounts:
- mountPath: /dbcmd-config/db.sh
name: {{ include "common.fullname" . }}-config
subPath: db.sh
command:
- /bin/sh
args:
- -x
- /dbcmd-config/db.sh
env:
- name: MYSQL_ROOT_PASSWORD
{{- include "common.secret.envFromSecretFast" (dict "global" . "uid" "db-root-password" "key" "password") | indent 10 }}
- name: MYSQL_HOST
value: "{{ index .Values "mariadb-galera" "service" "name" }}"
- name: MYSQL_USER
{{- include "common.secret.envFromSecretFast" (dict "global" . "uid" "db-secret" "key" "login") | indent 10 }}
- name: MYSQL_PORT
value: "{{ index .Values "mariadb-galera" "service" "internalPort" }}"
resources:
{{ include "common.resources" . }}
containers:
- name: {{ include "common.release" . }}-policy-galera-db-migrator
image: {{ .Values.repository }}/{{ .Values.dbmigrator.image }}
imagePullPolicy: {{ .Values.global.pullPolicy | default .Values.pullPolicy }}
volumeMounts:
- mountPath: /dbcmd-config/db_migrator_policy_init.sh
name: {{ include "common.fullname" . }}-config
subPath: db_migrator_policy_init.sh
command:
- /bin/sh
args:
- -x
- /dbcmd-config/db_migrator_policy_init.sh
env:
- name: SQL_HOST
value: "{{ index .Values "mariadb-galera" "service" "name" }}"
- name: SQL_USER
{{- include "common.secret.envFromSecretFast" (dict "global" . "uid" "db-secret" "key" "login") | indent 10 }}
- name: SQL_PASSWORD
{{- include "common.secret.envFromSecretFast" (dict "global" . "uid" "db-secret" "key" "password") | indent 10 }}
- name: SQL_DB
value: {{ .Values.dbmigrator.schema }}
- name: POLICY_HOME
value: {{ .Values.dbmigrator.policy_home }}
resources:
{{ include "common.resources" . }}
restartPolicy: Never
serviceAccountName: {{ include "common.fullname" (dict "suffix" "read" "dot" . )}}
volumes:
- name: {{ include "common.fullname" . }}-config
configMap:
name: {{ include "common.fullname" . }}-db-configmap
items:
- key: db.sh
path: db.sh
- key: db_migrator_policy_init.sh
path: db_migrator_policy_init.sh
Test Results
get pods
root@esy-master-policy-002-nfs:/home/kevint# kubectl get pods | grep policy
dev-policy-apex-pdp-0 1/1 Running 0 3h22m
dev-policy-api-6c4c87fd64-c5g6k 1/1 Running 0 17h
dev-policy-clamp-be-5c6cb758d8-cr7xc 1/1 Running 0 17h
dev-policy-clamp-fe-569494fb5d-4gdq2 1/1 Running 0 17h
dev-policy-clamp-galera-config-5d76k 0/1 Completed 0 17h
dev-policy-distribution-5b68646475-gjbq8 1/1 Running 0 17h
dev-policy-drools-pdp-0 1/1 Running 0 3h26m
dev-policy-galera-config-tt6mm 0/1 Completed 0 17h
dev-policy-mariadb-0 2/2 Running 0 17h
dev-policy-pap-65ddfc8f54-dmt6w 1/1 Running 0 17h
dev-policy-xacml-pdp-d7dfc9964-wn9zl 0/1 CrashLoopBackOff 266 17h
describe pod
root@esy-master-policy-002-nfs:/home/kevint# kubectl describe pod dev-policy-galera-config-tt6mm
Name: dev-policy-galera-config-tt6mm
Namespace: onap
Priority: 0
Node: esy-master-policy-002-k8s-02/172.16.1.104
Start Time: Thu, 19 Aug 2021 14:47:45 +0000
Labels: app=policy-job
controller-uid=e1875e15-c28e-4063-9d1c-787a4cff37ba
job-name=dev-policy-galera-config
release=dev
Annotations: cni.projectcalico.org/podIP:
cni.projectcalico.org/podIPs:
Status: Succeeded
IP: 10.42.2.11
IPs:
IP: 10.42.2.11
Controlled By: Job/dev-policy-galera-config
Init Containers:
policy-readiness:
Container ID: docker://c1d4ccf52662a3acf7750d97997b0949fc60e432aee480625ea45c230ae20853
Image: nexus3.onap.org:10001/onap/oom/readiness:3.0.1
Image ID: docker-pullable://nexus3.onap.org:10001/onap/oom/readiness@sha256:317c8a361ae73750f4d4a1b682c42b73de39083f73228dede31fd68b16c089db
Port: <none>
Host Port: <none>
Command:
/app/ready.py
--container-name
policy-mariadb
State: Terminated
Reason: Completed
Exit Code: 0
Started: Thu, 19 Aug 2021 14:48:06 +0000
Finished: Thu, 19 Aug 2021 14:48:47 +0000
Ready: True
Restart Count: 0
Environment:
NAMESPACE: onap (v1:metadata.namespace)
Mounts:
/var/run/secrets/kubernetes.io/serviceaccount from dev-policy-read-token-pdhhk (ro)
dev-policy-galera-config:
Container ID: docker://a6f5618cff66ec31b5352ce00b0a02c1063890be63d5be83a76ac3aac6eb6756
Image: docker.io/mariadb:10.5.8
Image ID: docker-pullable://mariadb@sha256:8040983db146f729749081c6b216a19d52e0973134e2e34c0b4fd87f48bc15b0
Port: <none>
Host Port: <none>
Command:
/bin/sh
Args:
-x
/dbcmd-config/db.sh
State: Terminated
Reason: Completed
Exit Code: 0
Started: Thu, 19 Aug 2021 14:49:26 +0000
Finished: Thu, 19 Aug 2021 14:49:27 +0000
Ready: True
Restart Count: 0
Environment:
MYSQL_ROOT_PASSWORD: <set to the key 'password' in secret 'dev-policy-db-root-password'> Optional: false
MYSQL_HOST: policy-mariadb
MYSQL_USER: <set to the key 'login' in secret 'dev-policy-db-secret'> Optional: false
MYSQL_PORT: 3306
Mounts:
/dbcmd-config/db.sh from dev-policy-config (rw,path="db.sh")
/var/run/secrets/kubernetes.io/serviceaccount from dev-policy-read-token-pdhhk (ro)
Containers:
dev-policy-galera-dbmigrator:
Container ID: docker://bf137b7dd920307e6a518c9ec7db1ae6502ea155161081754b44b217ca317960
Image: nexus3.onap.org:10001/onap/policy-db-migrator:2.3.0-SNAPSHOT
Image ID: docker-pullable://nexus3.onap.org:10001/onap/policy-db-migrator@sha256:59983c3fcf178335839448950033719effea48c1b39e78e5972972ab4965f4cc
Port: <none>
Host Port: <none>
Command:
/bin/sh
Args:
-x
/dbcmd-config/db_migrator_policy_init.sh
State: Terminated
Reason: Completed
Exit Code: 0
Started: Thu, 19 Aug 2021 14:50:09 +0000
Finished: Thu, 19 Aug 2021 14:50:19 +0000
Ready: False
Restart Count: 0
Environment:
SQL_HOST: policy-mariadb
SQL_USER: <set to the key 'login' in secret 'dev-policy-db-secret'> Optional: false
SQL_PASSWORD: <set to the key 'password' in secret 'dev-policy-db-secret'> Optional: false
SQL_DB: policyadmin
POLICY_HOME: /opt/app/policy
Mounts:
/dbcmd-config/db_migrator_policy_init.sh from dev-policy-config (rw,path="db_migrator_policy_init.sh")
/var/run/secrets/kubernetes.io/serviceaccount from dev-policy-read-token-pdhhk (ro)
Conditions:
Type Status
Initialized True
Ready False
ContainersReady False
PodScheduled True
Volumes:
dev-policy-config:
Type: ConfigMap (a volume populated by a ConfigMap)
Name: dev-policy-db-configmap
Optional: false
dev-policy-read-token-pdhhk:
Type: Secret (a volume populated by a Secret)
SecretName: dev-policy-read-token-pdhhk
Optional: false
QoS Class: BestEffort
Node-Selectors: <none>
Tolerations: node.kubernetes.io/not-ready:NoExecute op=Exists for 300s
node.kubernetes.io/unreachable:NoExecute op=Exists for 300s
Events: <none>
get secrets
/home/kevint# kubectl get secrets | grep policy-db
dev-policy-db-root-password Opaque 1 17h
dev-policy-db-secret Opaque 2 17h
retrieve secret
kubectl get secret dev-policy-db-secret -o jsonpath='{.data}'
{"login":"cG9saWN5X3VzZXI=","password":"U3VqaVJvaGI2LERvbnU="}root@esy-master-policy-002-nfs:~#
root@esy-master-policy-002-nfs:~# echo "cG9saWN5X3VzZXI=" | base64 --decode
policy_user
root@esy-master-policy-002-nfs:~# echo "U3VqaVJvaGI2LERvbnU=" | base64 --decode
policy_user_secret
login to database
kubectl exec -it dev-policy-mariadb-0 -- sh
Defaulting container name to policy-mariadb.
Use 'kubectl describe pod/dev-policy-mariadb-0 -n onap' to see all of the containers in this pod.
$ mysql -upolicy_user -p*********
Welcome to the MariaDB monitor. Commands end with ; or \g.
Your MariaDB connection id is 25531
Server version: 10.5.8-MariaDB-log Source distribution
Copyright (c) 2000, 2018, Oracle, MariaDB Corporation Ab and others.
Type 'help;' or '\h' for help. Type '\c' to clear the current input statement.
MariaDB [(none)]> use policyadmin;
Reading table information for completion of table and column names
You can turn off this feature to get a quicker startup with -A
Database changed
MariaDB [policyadmin]> show tables;
+------------------------------------------------------+
| Tables_in_policyadmin |
+------------------------------------------------------+
| jpapdpgroup_properties |
| jpapdpstatistics_enginestats |
| jpapdpsubgroup_policies |
| jpapdpsubgroup_properties |
| jpapdpsubgroup_supportedpolicytypes |
| jpatoscacapabilityassignment_attributes |
| jpatoscacapabilityassignment_metadata |
| jpatoscacapabilityassignment_occurrences |
| jpatoscacapabilityassignment_properties |
| jpatoscacapabilitytype_metadata |
| jpatoscacapabilitytype_properties |
| jpatoscadatatype_constraints |
| jpatoscadatatype_metadata |
| jpatoscadatatype_properties |
| jpatoscanodetemplate_metadata |
| jpatoscanodetemplate_properties |
| jpatoscanodetype_metadata |
| jpatoscanodetype_properties |
| jpatoscapolicy_metadata |
| jpatoscapolicy_properties |
| jpatoscapolicy_targets |
| jpatoscapolicytype_metadata |
| jpatoscapolicytype_properties |
| jpatoscapolicytype_targets |
| jpatoscapolicytype_triggers |
| jpatoscaproperty_constraints |
| jpatoscaproperty_metadata |
| jpatoscarelationshiptype_metadata |
| jpatoscarelationshiptype_properties |
| jpatoscarequirement_metadata |
| jpatoscarequirement_occurrences |
| jpatoscarequirement_properties |
| jpatoscaservicetemplate_metadata |
| jpatoscatopologytemplate_inputs |
| pdp |
| pdpgroup |
| pdpgroup_pdpsubgroup |
| pdppolicystatus |
| pdpstatistics |
| pdpsubgroup |
| pdpsubgroup_pdp |
| toscacapabilityassignment |
| toscacapabilityassignments |
| toscacapabilityassignments_toscacapabilityassignment |
| toscacapabilitytype |
| toscacapabilitytypes |
| toscacapabilitytypes_toscacapabilitytype |
| toscadatatype |
| toscadatatypes |
| toscadatatypes_toscadatatype |
| toscanodetemplate |
| toscanodetemplates |
| toscanodetemplates_toscanodetemplate |
| toscanodetype |
| toscanodetypes |
| toscanodetypes_toscanodetype |
| toscaparameter |
| toscapolicies |
| toscapolicies_toscapolicy |
| toscapolicy |
| toscapolicytype |
| toscapolicytypes |
| toscapolicytypes_toscapolicytype |
| toscaproperty |
| toscarelationshiptype |
| toscarelationshiptypes |
| toscarelationshiptypes_toscarelationshiptype |
| toscarequirement |
| toscarequirements |
| toscarequirements_toscarequirement |
| toscaservicetemplate |
| toscatopologytemplate |
| toscatrigger |
+------------------------------------------------------+
73 rows in set (0.011 sec)
MariaDB [policyadmin]> use migration;
Reading table information for completion of table and column names
You can turn off this feature to get a quicker startup with -A
Database changed
MariaDB [migration]> show tables;
+----------------------------+
| Tables_in_migration |
+----------------------------+
| db_metadata_versions |
| metadata_versions |
| policyadmin_change_history |
| pooling_history |
+----------------------------+
4 rows in set (0.001 sec)
MariaDB [migration]> select * from policyadmin_change_history order by id asc;
+----+---------------------------------------+-----------+--------------+------------+-------------------+---------+---------------------+
| ID | script | operation | from_version | to_version | tag | success | atTime
|
+----+---------------------------------------+-----------+--------------+------------+-------------------+---------+---------------------+
| 1 | 0100-jpapdpstatistics_enginestats.sql | upgrade | 0800 | 0900 | 1908211450100900u | 1 | 2021-08-19 14:50:13 |
| 2 | 0110-jpapdpstatistics_enginestats.sql | upgrade | 0800 | 0900 | 1908211450100900u | 1 | 2021-08-19 14:50:13 |
| 3 | 0120-pdp.sql | upgrade | 0800 | 0900 | 1908211450100900u | 1 | 2021-08-19 14:50:13 |
| 4 | 0130-idx_tsidx1.sql | upgrade | 0800 | 0900 | 1908211450100900u | 1 | 2021-08-19 14:50:14 |
| 5 | 0140-pk_pdpstatistics.sql | upgrade | 0800 | 0900 | 1908211450100900u | 1 | 2021-08-19 14:50:16 |
| 6 | 0150-pdpstatistics.sql | upgrade | 0800 | 0900 | 1908211450100900u | 1 | 2021-08-19 14:50:17 |
| 7 | 0160-pk_pdpstatistics.sql | upgrade | 0800 | 0900 | 1908211450100900u | 0 | 2021-08-19 14:50:17 |
+----+---------------------------------------+-----------+--------------+------------+-------------------+---------+---------------------+
7 rows in set (0.728 sec)
Upgrade failed
Manual fix:
update pdpstatistics as p JOIN (select name, version, timeStamp, ROW_NUMBER() OVER (ORDER BY timeStamp ASC) AS row_num from pdpstatistics GROUP BY name, version, timeStamp) as t ON (p.name=t.name and p.version=t.version and p.timeStamp = t.timeStamp) set
p.id=t.row_num;
Deployment 2:
+ /opt/app/policy/bin/prepare_upgrade.sh policyadmin
278 blocks
Preparing upgrade release version: 0800
Preparing upgrade release version: 0900
Done
+ /opt/app/policy/bin/db-migrator -s policyadmin -o upgrade
upgrade: 0900 -> 0900
Partial upgrade detected
Upgrade will attempt to resume at 0160-pk_pdpstatistics.sql
> upgrade 0160-pk_pdpstatistics.sql
--------------
ALTER TABLE pdpstatistics ADD CONSTRAINT PK_PDPSTATISTICS PRIMARY KEY (ID, name, version)
--------------
> upgrade 0170-pdpstatistics.sql
--------------
ALTER TABLE pdpstatistics MODIFY COLUMN timeStamp datetime NULL
--------------
> upgrade 0180-jpapolicyaudit.sql
--------------
CREATE TABLE IF NOT EXISTS jpapolicyaudit (ACTION INT DEFAULT NULL NULL, PDPGROUP VARCHAR(255) NULL, PDPTYPE VARCHAR(255) NULL, TIMESTAMP datetime DEFAULT NULL NULL, USER VARCHAR(255) NULL, ID BIGINT NOT NULL, name VARCHAR(120) NOT NULL, version VARCHAR(20) NOT NULL, CONSTRAINT PK_JPAPOLICYAUDIT PRIMARY KEY (ID, name, version))
--------------
> upgrade 0190-JpaPolicyAuditIndex_timestamp.sql
--------------
CREATE INDEX JpaPolicyAuditIndex_timestamp ON jpapolicyaudit(TIMESTAMP)
--------------
> upgrade 0200-sequence.sql
--------------
CREATE TABLE IF NOT EXISTS sequence (SEQ_NAME VARCHAR(50) NOT NULL, SEQ_COUNT DECIMAL(38) DEFAULT NULL NULL, CONSTRAINT PK_SEQUENCE PRIMARY KEY (SEQ_NAME))
--------------
> upgrade 0210-sequence.sql
--------------
INSERT INTO sequence(SEQ_NAME, SEQ_COUNT) VALUES('SEQ_GEN', 0)
--------------
policyadmin: OK: upgrade (0900)
+ /opt/app/policy/bin/db-migrator -s policyadmin -o report
name version
policyadmin 0900
ID script operation from_version to_version tag success atTime
1 0100-jpapdpstatistics_enginestats.sql upgrade 0800 0900 1908211450100900u 1 2021-08-19 14:50:13
2 0110-jpapdpstatistics_enginestats.sql upgrade 0800 0900 1908211450100900u 1 2021-08-19 14:50:13
3 0120-pdp.sql upgrade 0800 0900 1908211450100900u 1 2021-08-19 14:50:13
4 0130-idx_tsidx1.sql upgrade 0800 0900 1908211450100900u 1 2021-08-19 14:50:14
5 0140-pk_pdpstatistics.sql upgrade 0800 0900 1908211450100900u 1 2021-08-19 14:50:16
6 0150-pdpstatistics.sql upgrade 0800 0900 1908211450100900u 1 2021-08-19 14:50:17
7 0160-pk_pdpstatistics.sql upgrade 0800 0900 1908211450100900u 0 2021-08-19 14:50:17
8 0160-pk_pdpstatistics.sql upgrade 0800 0900 2008211234230900u 1 2021-08-20 12:34:25
9 0170-pdpstatistics.sql upgrade 0800 0900 2008211234230900u 1 2021-08-20 12:34:26
10 0180-jpapolicyaudit.sql upgrade 0800 0900 2008211234230900u 1 2021-08-20 12:34:26
11 0190-JpaPolicyAuditIndex_timestamp.sql upgrade 0800 0900 2008211234230900u 1 2021-08-20 12:34:27
12 0200-sequence.sql upgrade 0800 0900 2008211234230900u 1 2021-08-20 12:34:27
13 0210-sequence.sql upgrade 0800 0900 2008211234230900u 1 2021-08-20 12:34:27
policyadmin: OK @ 0900
root@esy-master-policy-002-nfs:~/oom/kubernetes#