By Philip Blackwood , AT&T
This document outlines how to use historical inventory and topology data to support various use cases. The purpose of the exercise is to ensure that all needed capabilities have been identified, and close any gaps discovered.
...
The functions below are written in the Clojure language to align with the historical data feature in ONAP (work in progress, snapshot taken 6/17/2019, master copy is in file “time filters …”).
| Code Block | ||||
|---|---|---|---|---|
| ||||
; time filters written in Clojure language, using a flat data structure
; logic as documented in white paper
; the different filters can be combined, or invoked interactively
; also see listing of use cases
; -------------------------------------------------------------------------------------
; terminology
;
; a "value" is one instance of a value for an attribute (one row of text in the data definition below)
; a value has a timestamp for its startTime and one for its WriteTime
; a "view" consists of the data that is known at a time T (i.e. written before T)
;
; names of filters start with either "getValues..." or "filterOutValues ..." (whichever style is easier to understand)
; the phrase "NotNeeded" means "not needed for any view back to time T"
;
; ---------------------------------------------------------------------------------------
; List of Main Functions
;
;
; Prepare an initial data set that can then be used to answer questions
;
; getBaseDataSetForT
; getConnectedSubgraph (uses mergeIntersectingSets and intersecting?)
;
;
; Topology and Entity Level Views
;
; getTopology ... AfterTimeT, BeforeTimeT, BetweenTimes, or AtTimeT
;
;
; History of attribute values
;
; getHistory ... AfterTimeT, BeforeTimeT, BetweenTImes, or AtTimeT
;
; getTopology ... AfterTimeT, BeforeTimeT, BetweenTimes, or AtTimeT
;
; getChanges ... AfterTimeT, BeforeTimeT, BetweenTimes, or AtTimeT
;
;
; Check for data missing or not valid in a view (e.g. arrived late)
;
; getValuesMissingAtTimeT
;
; getValuesMarkedNotValidAfterTimeT
;
;
; Identify older values to be removed (or marked in data base to improve query performance)
;
; getEntitiesNotNeeded
;
; getValuesNotNeeded
;
;
; Restrict the view to data that was available at time T
;
; getValuesWrittenBeforeTimeT
;
;
; Other
;
; getValuesForAttribute
; getValuesForAttributeList
; getValuesForAttributeAndValue
;
; getValuesMarkedNotValid
;
; filterOutEntitiesGoneBeforeTimeT
;
; findLagTimesAtT
; ------------------------------------------------------------------------------------------
; to do
;
;
; 1. make sure conditions for valid value are included where needed (e.g. exists = no, and valid)
;
; 2. replace integer times with actual time values, and change all time comparisons and calculations
;
; 3. test the functions that have a note saying not tested
; -------------------------------------------------------------------------------------------
; a flat data structure was chosen for ease of coding (e.g. do query and then format like this)
; functions to view the data do not assume any particular order of the entries
; every row of text below should have the same keywords; empty values should be represented as ""
; terminology note: in the data set an "entity" can be either a node or relationship
(def entities '(
{:entity "1111" :attribute "exists" :value "yes" :startTime 3 :writeTime 4 :timeMarkedNotValid "" }
{:entity "1111" :attribute "id" :value "1111" :startTime 3 :writeTime 4 :timeMarkedNotValid "" }
{:entity "1111" :attribute "entity-type" :value "node" :startTime 3 :writeTime 4 :timeMarkedNotValid "" }
{:entity "1111" :attribute "size" :value "M" :startTime 4 :writeTime 6 :timeMarkedNotValid "" }
{:entity "1111" :attribute "size" :value "XL" :startTime 10 :writeTime 25 :timeMarkedNotValid 35}
{:entity "1111" :attribute "in-maintenance" :value "yes" :startTime 8 :writeTime 8 :timeMarkedNotValid "" }
{:entity "1111" :attribute "in-maintenance" :value "no" :startTime 15 :writeTime 15 :timeMarkedNotValid "" }
{:entity "2222" :attribute "exists" :value "yes" :startTime 103 :writeTime 104 :timeMarkedNotValid "" }
{:entity "2222" :attribute "id" :value "111" :startTime 103 :writeTime 104 :timeMarkedNotValid "" }
{:entity "2222" :attribute "entity-type" :value "node" :startTime 103 :writeTime 103 :timeMarkedNotValid "" }
{:entity "2222" :attribute "size" :value "XL" :startTime 110 :writeTime 125 :timeMarkedNotValid "" }
{:entity "2222" :attribute "size" :value "M" :startTime 103 :writeTime 104 :timeMarkedNotValid "" }
{:entity "2222" :attribute "in-maintenance" :value "yes" :startTime 108 :writeTime 108 :timeMarkedNotValid "" }
{:entity "2222" :attribute "in-maintenance" :value "no" :startTime 115 :writeTime 115 :timeMarkedNotValid "" }
{:entity "2222" :attribute "exists" :value "no" :startTime 140 :writeTime 141 :timeMarkedNotValid "" }
{:entity "3333" :attribute "exists" :value "yes" :startTime 503 :writeTime 504 :timeMarkedNotValid "" }
{:entity "3333" :attribute "id " :value "3333" :startTime 503 :writeTime 504 :timeMarkedNotValid "" }
{:entity "3333" :attribute "entity-type " :value "edge" :startTime 503 :writeTime 504 :timeMarkedNotValid "" }
{:entity "3333" :attribute "from-entity" :value "1111" :startTime 503 :writeTime 504 :timeMarkedNotValid "" }
{:entity "3333" :attribute "to-entity " :value "2222" :startTime 503 :writeTime 504 :timeMarkedNotValid "" }
{:entity "3333" :attribute "exists" :value "no" :startTime 530 :writeTime 531 :timeMarkedNotValid "" }
))
; for testing when looking at values for one attribute
(def values '(
{:entity "1111" :attribute "size" :value "M" :startTime 4 :writeTime 5 :timeMarkedNotValid ""}
{:entity "1111" :attribute "size" :value "S" :startTime 12 :writeTime 13 :timeMarkedNotValid ""}
{:entity "1111" :attribute "size" :value "XL" :startTime 8 :writeTime 20 :timeMarkedNotValid ""}))
; =====================================================================================
;
; functions below are listed in a bottoms-up order
;
; =====================================================================================
; getHistoryAfterTimeT and supporting functions
(defn findAttributeLatestStartTimeBeforeTimeT
[T values]
(let [timesList (sort > (map :startTime values)) ; reverse sort the times
candidate (some #(if (<= % T ) %) timesList)] ; select first that is <= T
(if (nil? candidate) (last timesList) candidate ) )) ; or just return earliest
;
; example
; (findAttributeLatestStartTimeBeforeTimeT 1 values)
; not tested
(defn findAttributeValuesForHistoryAfterTimeT
"for one attribute, find values needed for history after time T, in current view"
[T values]
(let [startTimeCutoff (findAttributeLatestStartTimeBeforeTimeT T values)]
(filter #(>= (:startTime %) startTimeCutoff) values) ))
; example
; (findAttributeValuesForHistoryAfterTimeT 1 values)
; all tests passed
(defn applyTimeFilter
"apply an attribute-level filter to each attribute in a data set"
[timeFilter T dataSet]
(let [attributeList (map (juxt :entity :attribute) dataSet)]
(reduce (fn [result attribute]
(let [values (filter #(and (= (:entity %) (first attribute)) (= (:attribute %) (second attribute))) dataSet)
keepers (timeFilter T values) ]
(into result keepers))) [] attributeList) ))
; (applyTimeFilter findAttributeValuesForHistoryAfterTimeT 18 entities)
(defn findEntitiesGoneBeforeTimeT
"find entities gone before time T in the current view"
[T dataSet]
(->> dataSet
(filter #(= (:attribute %) "exists") ,,,)
(filter #(= (:value %) "no" ) ,,,)
(filter #(= (:timeMarkedNotValid %) "" ) ,,,)
(filter #(<= (:startTime %) T ) ,,,)
(map :entity ,,,) ))
; example
; (findEntitiesGoneBeforeTimeT 5 entities)
; all tests passed
(defn filterOutEntitiesGoneBeforeTimeT
"filter out entities that did exist at time T, in the current view"
[T dataSet]
(let [entityList (findEntitiesGoneBeforeTimeT T dataSet)]
(filter #(nil? (some #{(:entity %)} entityList)) dataSet) )) ;keep values for entities not on the list
; example
; (filterOutEntitiesGoneBeforeTimeT 5 entities)
; all tests passed
(defn getHistoryAfterTimeT
" filter to find entries needed to show history starting at time T"
[T dataSet]
(->> dataSet
(filterOutEntitiesGoneBeforeTimeT T ,,,)
(applyTimeFilter findAttributeValuesForHistoryAfterTimeT T ,,,) ))
; example
; (getHistoryAfterTimeT 6 entities)
; =============================================================================
; getBaseDataSetForT and supporting functions not yet listed
(defn filterOutAttributeValuesNotNeeded
"filter out values not needed for any view back to T, for one attribute"
[T values]
(let [startTimeCutoff (->> values
(filter #(< (:writeTime %) T) ,,,)
(findAttributeLatestStartTimeBeforeTimeT T ,,,) )]
(filter #(>= (:startTime %) startTimeCutoff) values) ))
; example
; (filterOutAttributeValuesNotNeeded 18 entities)
(defn filterOutValuesNotNeeded
"filter out values not needed for any view"
[T dataSet]
(applyTimeFilter filterOutAttributeValuesNotNeeded T dataSet) )
; (filterOutValuesNotNeeded 14 entities)
(defn findEntitiesNotNeeded
"find entities not needed for any view back to start time T"
[T dataSet]
(->> dataSet
(filter #(<= (:writeTime %) T) ,,,)
(findEntitiesGoneBeforeTimeT T ,,,) ))
; example
; (findEntitiesNotNeeded 25 entities)
(defn filterOutEntitiesNotNeeded
"filter out entities not needed for any view back to start time T"
[T dataSet]
(let [entityList (findEntitiesNotNeeded T dataSet)]
(filter #(nil? (some #{(:entity %)} entityList)) dataSet) )) ; keep values for entities not on the list
;
; example
; (filterOutEntitiesNotNeeded 300 entities)
(defn getBaseDataSetForT
"keep only the data needed to answer questions about history after time T (for any view)"
[T dataSet]
(->> dataSet
(filterOutEntitiesNotNeeded T ,,,)
(filterOutValuesNotNeeded T ,,,) ))
; ==========================================================================
; getConnectedSubgraph and supporting functions
(defn intersecting?
"check to see if the intersection of two sets is non-empty"
[set1 set2]
(not (empty? (clojure.set/intersection set1 set2))) )
; example
; (intersecting? #{1 2} #{3 4 5})
(defn mergeIntersectingSets
"from a given starting set and a collection of sets, find union of sets reachable by intersections"
[result-set sets]
(let [new-result-set (reduce #(if (intersecting? %1 %2) (clojure.set/union %1 %2) %1) result-set sets)]
(if (= result-set new-result-set)
result-set
(recur new-result-set sets)) ))
; example
; (def sets '( #{1 2} #{2 3} #{5 7} #{3 5} #{1 8 10} #{12 14 20}) )
; (mergeIntersectingSets #{10} sets)
(defn getConnectedSubgraph
"given a collection of pairs representing edges, find connected graph of startNode"
[startNode edges]
(let [nodes (mergeIntersectingSets #{startNode} edges)]
(filter #(and (contains? nodes (first %)) (contains? nodes (second %))) edges)))
; example
; (def edges '( #{1 3} #{8 9} #{21 22} #{4 5} #{3 4} #{4 8}))
; (getConnectedSubgraph 1 edges)
; ==========================================================================
; other functions in alphabetical order
; --------------------------------------------------------------------------
; filterOutValuesMarkedNotValid
(defn filterOutValuesMarkedNotValid
"filter out entries that have been marked not valid"
[dataSet]
(filter #(= (:timeMarkedNotValid %) "") entities) )
; --------------------------------------------------------------------------
; findAttributeValuesGoneBeforeTimeT
; might also need to account for case when the entity is gone before T
(defn findAttributeValuesGoneBeforeTimeT
"find values gone before T, for one attribute, current view"
[T values]
(let [startTimeCutoff (findAttributeLatestStartTimeBeforeTimeT T values)]
(filter #(< (:startTime %) startTimeCutoff) values) ))
; --------------------------------------------------------------------------
; findAttributeValuesNotNeeded (e.g. to archive and remove)
(defn findAttributeValuesNotNeeded
"find values not needed for any view back to T, for one attribute"
[T values]
(let [startTimeCutoff (->> values
(filter #(< (:writeTime %) T) ,,, )
(findAttributeLatestStartTimeBeforeTimeT T ,,,))]
(filter #(< (:startTime %) startTimeCutoff) values) ))
; --------------------------------------------------------------------------
; findLagTimesAtT
(defn findLagTimesAtT
"delay between a time T at which a value was in effect, and when known
Requires dataSet to consist of values in effect as of time T"
[T dataSet]
(->> dataSet
(sort-by #(:writeTime %) > ,,,)
(map #(- (:writeTime %) T) ,,, )))
; example
; (lagTimesAtT 20 entities)
; ----------------------------------------------------------------------
; getChangesAfterTimeT
(defn getChangesAfterTimeT
"filter to find entries with start time greater than or equal to T"
[T dataSet]
(filter #(>= (:startTime %) T ) dataSet))
; example
; (getChangesAfterTimeT 8 entities)
; ---------------------------------------------------------------------
; getChangesAtTimeT
; should replace this with getChangesNearTimeT and allow client to specify size of window around T
(defn getChangesAtTimeT
"filter to find entries with start time equal to T"
[T dataSet]
(filter #(= (:startTime %) T ) dataSet))
; ---------------------------------------------------------------------
; getChangesBeforeTimeT
(defn getChangesBeforeTimeT
"filter to find entries with start time less than or equal to T"
[T dataSet]
(filter #(<= (:startTime %) T ) dataSet))
; example
; (getChangesBeforeTimeT 8 entities)
; ---------------------------------------------------------------------
; getChangesBetweenTimes
(defn getHistoryBeforeTimeT
"filter to find entries with start time less than or equal to T"
[T dataSet]
(filter #(<= (:startTime %) T) dataSet) )
(defn getChangesBetweenTimes
"filter to find entries with start time between T1 and T2 (inclusive)"
[T1 T2 dataSet]
(->> dataSet
(getChangesAfterTimeT T1 ,,,)
(getHistoryBeforeTimeT T2 ,,,) ))
; -----------------------------------------------------------------------
; getHistoryAtTimeT
(defn getHistoryAtTimeT
"get values at time T."
[T dataSet]
(->> dataSet
(getHistoryAfterTimeT T ,,,)
(getHistoryBeforeTimeT T ,,, ) ))
; ----------------------------------------------------------------------
; getHistoryBeforeTimeT ... see getChangesBetweenTimes
; ----------------------------------------------------------------------
; getHistoryBetweenTimes
(defn getHistoryBetweenTimes
"filter to find entries showing values between two times"
[T1 T2 dataSet]
(->> dataSet
(getHistoryAfterTimeT T1 ,,,)
(getHistoryBeforeTimeT T2 ,,,) ))
; ----------------------------------------------------------------------
; getTopologyAtTimeT
; entities that exist at time T
; ---------------------------------------------------------------------
; getValuesForAttribute
; getValuesForAttributeAndValue
; getValuesForAttributeList
(defn getValuesForAttribute
"filter to find entries for a single attribute"
[attribute dataSet]
(filter #(= (:attribute %) attribute) dataSet) )
; example
; (getValuesForAttribute "size" entities)
(defn getValuesForAttributeAndValue
"filter to find entries for a specific attribute and value"
[attribute value dataSet]
(->> dataSet
(filter #(= (:attribute %) attribute) ,,,)
(filter #(= (:value %) value ) ,,,)))
; example
; (getValuesForAttributeAndValue "size" "XL" entities)
(defn getValuesForAttributeList
"filter to find entries for a list of attributes"
[attributeList dataSet]
(filter #(some #{(:attribute %)} attributeList) dataSet) )
; example
; (getValuesForAttributeList '("size" "in-maintenance") entities)
; ---------------------------------------------------------------------
; getValuesForEntityExistence (rename as a topology filter)
(defn getValuesForEntityExistence
"filter to find entries that indicate an entity was created or removed"
[dataSet]
(getValuesForAttribute "exists" dataSet))
; ---------------------------------------------------------------------------------
; getValuesMissingAtTimeT
(defn getValuesWrittenAfterTimeT
"filter to find entries that were not known as of time T"
[T dataSet]
(filter #(> (:writeTime %) T) dataSet))
(defn getValuesMissingAtTimeT
"find values that were delayed in getting written, with startTime before T but written after T"
[T dataSet]
(->> dataSet
(getHistoryBeforeTimeT T ,,,)
(getValuesWrittenAfterTimeT T ,,,) ))
; ---------------------------------------------------------------------------------
; getValuesMarkedNotValidAfterTimeT (e.g. data used prior to time T that was later marked not valid)
(defn getValuesWrittenBeforeTimeT
"filter to find entries that were known as of time T"
[T dataSet]
(filter #(<= (:writeTime %) T) dataSet))
(defn getValuesMarkedNotValid
"filter to find entries that have been marked not valid"
[dataSet]
(filter #(not= (:timeMarkedNotValid %) "") entities) )
(defn getValuesMarkedNotValidAfterTimeT
"find values with a startTime before T and a value of timeMarkedNotValid later than T"
[T dataSet]
(->> dataSet
(getValuesWrittenBeforeTimeT T ,,,)
(getValuesMarkedNotValid ,,,)
(filter #(> (:timeMarkedNotValid %) T) ,,,) ))
; ---------------------------------------------------------------------------------
; getValuesNotNeeded (has not been tested)
(defn getValuesNotNeeded
"filter to find values not needed for any view back to time T"
[T dataSet]
(applyTimeFilter findAttributeValuesNotNeeded T dataSet) )
; ---------------------------------------------------------------------------------
; getValuesWrittenAfterTimeT ... see getValuesMissingAtTimeT
; ---------------------------------------------------------------------------------
; getValuesWrittenBeforeTimeT ... see getValuesMarkedNotValidAfterTimeT
; ---------------------------------------------------------------------------------
; sortByValueStartTimes
(defn sortByValueStartTimes
"sort by entity, attribute, and start time of values of the attribute"
[dataSet]
(sort-by (juxt :entity :attribute :startTime) dataSet))
; -------- end of listing ---------------
|
...