building out tests for extraction functions.

Same functions will power every other model. Found some opprotunities to add control over circular references, when and how to fire errors and what not. All in all, some robust code additions will improve the library significantly.

datamodel/low/extraction_functions.go

@@ -8,6 +8,7 @@ import (
 	"github.com/pb33f/libopenapi/index"
 	"github.com/pb33f/libopenapi/utils"
 	"github.com/vmware-labs/yaml-jsonpath/pkg/yamlpath"
+	"go.uber.org/zap"
 	"gopkg.in/yaml.v3"
 	"strconv"
 	"strings"
@@ -22,7 +23,7 @@ func FindItemInMap[T any](item string, collection map[KeyReference[string]]Value
 	return nil
 }
 
-func LocateRefNode(root *yaml.Node, idx *index.SpecIndex) *yaml.Node {
+func LocateRefNode(root *yaml.Node, idx *index.SpecIndex) (*yaml.Node, error) {
 	if rf, _, rv := utils.IsNodeRefValue(root); rf {
 		// run through everything and return as soon as we find a match.
 		// this operates as fast as possible as ever
@@ -69,7 +70,30 @@ func LocateRefNode(root *yaml.Node, idx *index.SpecIndex) *yaml.Node {
 		for _, collection := range collections {
 			found = collection()
 			if found != nil && found[rv] != nil {
-				return found[rv].Node
+
+				// if this is a ref node, we need to keep diving
+				// until we hit something that isn't a ref.
+				if jh, _, _ := utils.IsNodeRefValue(found[rv].Node); jh {
+
+					// if this node is circular, stop drop and roll.
+					if !IsCircular(found[rv].Node, idx) {
+						return LocateRefNode(found[rv].Node, idx)
+					} else {
+						Log.Error("circular reference found during lookup, and will remain un-resolved.",
+							zap.Int("line", found[rv].Node.Line),
+							zap.Int("column", found[rv].Node.Column),
+							zap.String("reference", found[rv].Definition),
+							zap.String("journey",
+								GetCircularReferenceResult(found[rv].Node, idx).GenerateJourneyPath()))
+
+						return found[rv].Node, fmt.Errorf("circular reference '%s' found during lookup at line %d, column %d, "+
+							"It cannot be resolved",
+							GetCircularReferenceResult(found[rv].Node, idx).GenerateJourneyPath(),
+							found[rv].Node.Line,
+							found[rv].Node.Column)
+					}
+				}
+				return found[rv].Node, nil
 			}
 		}
 
@@ -83,23 +107,28 @@ func LocateRefNode(root *yaml.Node, idx *index.SpecIndex) *yaml.Node {
 			nodes, fErr := path.Find(idx.GetRootNode())
 			if fErr == nil {
 				if len(nodes) > 0 {
-					return nodes[0]
+					return nodes[0], nil
 				}
 			}
 		}
+		return nil, fmt.Errorf("reference '%s' at line %d, column %d was not found", root.Value, root.Line, root.Column)
 	}
-	return nil
+	return nil, nil
 }
 
 func ExtractObjectRaw[T Buildable[N], N any](root *yaml.Node, idx *index.SpecIndex) (T, error) {
-
+	var circError error
 	if h, _, _ := utils.IsNodeRefValue(root); h {
-		ref := LocateRefNode(root, idx)
+		ref, err := LocateRefNode(root, idx)
 		if ref != nil {
 			root = ref
+			if err != nil {
+				circError = err
+			}
 		} else {
-			return nil, fmt.Errorf("object extraction failed: reference cannot be found: %s, line %d, col %d",
-				root.Content[1].Value, root.Content[1].Line, root.Content[1].Column)
+			if err != nil {
+				return nil, fmt.Errorf("object extraciton failed: %s", err.Error())
+			}
 		}
 	}
 	var n T = new(N)
@@ -111,31 +140,42 @@ func ExtractObjectRaw[T Buildable[N], N any](root *yaml.Node, idx *index.SpecInd
 	if err != nil {
 		return n, err
 	}
+	if circError != nil && !idx.AllowCircularReferenceResolving() {
+		return n, circError
+	}
 	return n, nil
 }
 
 func ExtractObject[T Buildable[N], N any](label string, root *yaml.Node, idx *index.SpecIndex) (NodeReference[T], error) {
 	var ln, vn *yaml.Node
+	var circError error
 	if rf, rl, _ := utils.IsNodeRefValue(root); rf {
-		// locate reference in index.
-		ref := LocateRefNode(root, idx)
+		ref, err := LocateRefNode(root, idx)
 		if ref != nil {
 			vn = ref
 			ln = rl
+			if err != nil {
+				circError = err
+			}
 		} else {
-			return NodeReference[T]{}, fmt.Errorf("object build failed: reference cannot be found: %s",
-				root.Content[1].Value)
+			if err != nil {
+				return NodeReference[T]{}, fmt.Errorf("object extraciton failed: %s", err.Error())
+			}
 		}
 	} else {
 		_, ln, vn = utils.FindKeyNodeFull(label, root.Content)
 		if vn != nil {
 			if h, _, _ := utils.IsNodeRefValue(vn); h {
-				ref := LocateRefNode(vn, idx)
+				ref, lerr := LocateRefNode(vn, idx)
 				if ref != nil {
 					vn = ref
+					if lerr != nil {
+						circError = lerr
+					}
 				} else {
-					return NodeReference[T]{}, fmt.Errorf("object build failed: reference cannot be found: %s",
-						vn.Content[1].Value)
+					if lerr != nil {
+						return NodeReference[T]{}, fmt.Errorf("object extraciton failed: %s", lerr.Error())
+					}
 				}
 			}
 		}
@@ -152,22 +192,29 @@ func ExtractObject[T Buildable[N], N any](label string, root *yaml.Node, idx *in
 	if err != nil {
 		return NodeReference[T]{}, err
 	}
-	return NodeReference[T]{
+	res := NodeReference[T]{
 		Value:     n,
 		KeyNode:   ln,
 		ValueNode: vn,
-	}, nil
+	}
+	if circError != nil && !idx.AllowCircularReferenceResolving() {
+		return res, err
+	}
+	return res, nil
 }
 
 func ExtractArray[T Buildable[N], N any](label string, root *yaml.Node, idx *index.SpecIndex) ([]ValueReference[T],
 	*yaml.Node, *yaml.Node, error) {
 	var ln, vn *yaml.Node
+	var circError error
 	if rf, rl, _ := utils.IsNodeRefValue(root); rf {
-		// locate reference in index.
-		ref := LocateRefNode(root, idx)
+		ref, err := LocateRefNode(root, idx)
 		if ref != nil {
 			vn = ref
 			ln = rl
+			if err != nil {
+				circError = err
+			}
 		} else {
 			return []ValueReference[T]{}, nil, nil, fmt.Errorf("array build failed: reference cannot be found: %s",
 				root.Content[1].Value)
@@ -176,12 +223,17 @@ func ExtractArray[T Buildable[N], N any](label string, root *yaml.Node, idx *ind
 		_, ln, vn = utils.FindKeyNodeFull(label, root.Content)
 		if vn != nil {
 			if h, _, _ := utils.IsNodeRefValue(vn); h {
-				ref := LocateRefNode(vn, idx)
+				ref, err := LocateRefNode(vn, idx)
 				if ref != nil {
 					vn = ref
+					if err != nil {
+						circError = err
+					}
 				} else {
-					return []ValueReference[T]{}, nil, nil, fmt.Errorf("array build failed: reference cannot be found: %s",
-						root.Content[1].Value)
+					if err != nil {
+						return []ValueReference[T]{}, nil, nil, fmt.Errorf("array build failed: reference cannot be found: %s",
+							err.Error())
+					}
 				}
 			}
 		}
@@ -193,12 +245,17 @@ func ExtractArray[T Buildable[N], N any](label string, root *yaml.Node, idx *ind
 		}
 		for _, node := range vn.Content {
 			if rf, _, _ := utils.IsNodeRefValue(node); rf {
-				ref := LocateRefNode(node, idx)
+				ref, err := LocateRefNode(node, idx)
 				if ref != nil {
 					node = ref
+					if err != nil {
+						circError = err
+					}
 				} else {
-					return []ValueReference[T]{}, nil, nil, fmt.Errorf("array build failed: reference cannot be found: %s",
-						root.Content[1].Value)
+					if err != nil {
+						return []ValueReference[T]{}, nil, nil, fmt.Errorf("array build failed: reference cannot be found: %s",
+							err.Error())
+					}
 				}
 			}
 			var n T = new(N)
@@ -216,6 +273,9 @@ func ExtractArray[T Buildable[N], N any](label string, root *yaml.Node, idx *ind
 			})
 		}
 	}
+	if circError != nil && !idx.AllowCircularReferenceResolving() {
+		return items, ln, vn, circError
+	}
 	return items, ln, vn, nil
 }
 
@@ -236,6 +296,7 @@ func ExtractExample(expNode, expLabel *yaml.Node) NodeReference[any] {
 
 func ExtractMapFlatNoLookup[PT Buildable[N], N any](root *yaml.Node, idx *index.SpecIndex) (map[KeyReference[string]]ValueReference[PT], error) {
 	valueMap := make(map[KeyReference[string]]ValueReference[PT])
+	var circError error
 	if utils.IsNodeMap(root) {
 		var currentKey *yaml.Node
 		skip := false
@@ -254,11 +315,16 @@ func ExtractMapFlatNoLookup[PT Buildable[N], N any](root *yaml.Node, idx *index.
 			}
 			// if value is a reference, we have to look it up in the index!
 			if h, _, _ := utils.IsNodeRefValue(node); h {
-				ref := LocateRefNode(node, idx)
+				ref, err := LocateRefNode(node, idx)
 				if ref != nil {
 					node = ref
+					if err != nil {
+						circError = err
+					}
 				} else {
-					return nil, fmt.Errorf("map build failed: reference cannot be found: %s", root.Content[1].Value)
+					if err != nil {
+						return nil, fmt.Errorf("map build failed: reference cannot be found: %s", err.Error())
+					}
 				}
 			}
 
@@ -280,6 +346,9 @@ func ExtractMapFlatNoLookup[PT Buildable[N], N any](root *yaml.Node, idx *index.
 			}
 		}
 	}
+	if circError != nil && !idx.AllowCircularReferenceResolving() {
+		return valueMap, circError
+	}
 	return valueMap, nil
 }
 
@@ -290,12 +359,16 @@ type mappingResult[T any] struct {
 
 func ExtractMapFlat[PT Buildable[N], N any](label string, root *yaml.Node, idx *index.SpecIndex) (map[KeyReference[string]]ValueReference[PT], *yaml.Node, *yaml.Node, error) {
 	var labelNode, valueNode *yaml.Node
+	var circError error
 	if rf, rl, _ := utils.IsNodeRefValue(root); rf {
 		// locate reference in index.
-		ref := LocateRefNode(root, idx)
+		ref, err := LocateRefNode(root, idx)
 		if ref != nil {
 			valueNode = ref
 			labelNode = rl
+			if err != nil {
+				circError = err
+			}
 		} else {
 			return nil, labelNode, valueNode, fmt.Errorf("map build failed: reference cannot be found: %s",
 				root.Content[1].Value)
@@ -304,12 +377,17 @@ func ExtractMapFlat[PT Buildable[N], N any](label string, root *yaml.Node, idx *
 		_, labelNode, valueNode = utils.FindKeyNodeFull(label, root.Content)
 		if valueNode != nil {
 			if h, _, _ := utils.IsNodeRefValue(valueNode); h {
-				ref := LocateRefNode(valueNode, idx)
+				ref, err := LocateRefNode(valueNode, idx)
 				if ref != nil {
 					valueNode = ref
+					if err != nil {
+						circError = err
+					}
 				} else {
-					return nil, labelNode, valueNode, fmt.Errorf("map build failed: reference cannot be found: %s",
-						root.Content[1].Value)
+					if err != nil {
+						return nil, labelNode, valueNode, fmt.Errorf("map build failed: reference cannot be found: %s",
+							err.Error())
+					}
 				}
 			}
 		}
@@ -349,12 +427,17 @@ func ExtractMapFlat[PT Buildable[N], N any](label string, root *yaml.Node, idx *
 			}
 			// check our valueNode isn't a reference still.
 			if h, _, _ := utils.IsNodeRefValue(en); h {
-				ref := LocateRefNode(en, idx)
+				ref, err := LocateRefNode(en, idx)
 				if ref != nil {
 					en = ref
+					if err != nil {
+						circError = err
+					}
 				} else {
-					return nil, labelNode, valueNode, fmt.Errorf("flat map build failed: reference cannot be found: %s",
-						root.Content[1].Value)
+					if err != nil {
+						return nil, labelNode, valueNode, fmt.Errorf("flat map build failed: reference cannot be found: %s",
+							err.Error())
+					}
 				}
 			}
 
@@ -377,85 +460,12 @@ func ExtractMapFlat[PT Buildable[N], N any](label string, root *yaml.Node, idx *
 		}
 		return valueMap, labelNode, valueNode, nil
 	}
+	if circError != nil && !idx.AllowCircularReferenceResolving() {
+		return nil, labelNode, valueNode, circError
+	}
 	return nil, labelNode, valueNode, nil
 }
 
-func ExtractMap[PT Buildable[N], N any](label string, root *yaml.Node, idx *index.SpecIndex) (map[KeyReference[string]]map[KeyReference[string]]ValueReference[PT], error) {
-	var labelNode, valueNode *yaml.Node
-	if rf, rl, _ := utils.IsNodeRefValue(root); rf {
-		// locate reference in index.
-		ref := LocateRefNode(root, idx)
-		if ref != nil {
-			valueNode = ref
-			labelNode = rl
-		} else {
-			return nil, fmt.Errorf("map build failed: reference cannot be found: %s", root.Content[1].Value)
-		}
-	} else {
-		_, labelNode, valueNode = utils.FindKeyNodeFull(label, root.Content)
-		if valueNode != nil {
-			if h, _, _ := utils.IsNodeRefValue(valueNode); h {
-				ref := LocateRefNode(valueNode, idx)
-				if ref != nil {
-					valueNode = ref
-				} else {
-					return nil, fmt.Errorf("map build failed: reference cannot be found: %s",
-						root.Content[1].Value)
-				}
-			}
-		}
-	}
-
-	if valueNode != nil {
-		var currentLabelNode *yaml.Node
-		valueMap := make(map[KeyReference[string]]ValueReference[PT])
-		for i, en := range valueNode.Content {
-			if i%2 == 0 {
-				currentLabelNode = en
-				continue
-			}
-			if strings.HasPrefix(strings.ToLower(currentLabelNode.Value), "x-") {
-				continue // yo, don't pay any attention to extensions, not here anyway.
-			}
-
-			// check our valueNode isn't a reference still.
-			if h, _, _ := utils.IsNodeRefValue(en); h {
-				ref := LocateRefNode(en, idx)
-				if ref != nil {
-					en = ref
-				} else {
-					return nil, fmt.Errorf("map build failed: reference cannot be found: %s",
-						root.Content[1].Value)
-				}
-			}
-
-			var n PT = new(N)
-			err := BuildModel(en, n)
-			if err != nil {
-				return nil, err
-			}
-			berr := n.Build(en, idx)
-			if berr != nil {
-				return nil, berr
-			}
-			valueMap[KeyReference[string]{
-				Value:   currentLabelNode.Value,
-				KeyNode: currentLabelNode,
-			}] = ValueReference[PT]{
-				Value:     n,
-				ValueNode: en,
-			}
-		}
-		resMap := make(map[KeyReference[string]]map[KeyReference[string]]ValueReference[PT])
-		resMap[KeyReference[string]{
-			Value:   labelNode.Value,
-			KeyNode: labelNode,
-		}] = valueMap
-		return resMap, nil
-	}
-	return nil, nil
-}
-
 func ExtractExtensions(root *yaml.Node) map[KeyReference[string]]ValueReference[any] {
 	extensions := utils.FindExtensionNodes(root.Content)
 	extensionMap := make(map[KeyReference[string]]ValueReference[any])