(feat): Added Reference tracking to low-level model #25

When building a document, everything that IS NOT a schema is auto-resolved in the model, this is very convenient because it creates a nice tree to explore and there is no need to perform lookups to when using `$ref` instead of inline definitions.

The issue is however being able to determine if the node was originally a reference or not, that data was lost, including the name of the reference used. This use case surfaced in issue #25, where the need to know what is and what is not referenced has different requirements for different applications.

This update now tracks that data and makes it available in `NodeReference` and `ValueReference` for every property.

Signed-off-by: Dave Shanley <dave@quobix.com>

datamodel/low/extraction_functions.go

@@ -111,35 +111,39 @@ func LocateRefNode(root *yaml.Node, idx *index.SpecIndex) (*yaml.Node, error) {
 
 // ExtractObjectRaw will extract a typed Buildable[N] object from a root yaml.Node. The 'raw' aspect is
 // that there is no NodeReference wrapper around the result returned, just the raw object.
-func ExtractObjectRaw[T Buildable[N], N any](root *yaml.Node, idx *index.SpecIndex) (T, error) {
+func ExtractObjectRaw[T Buildable[N], N any](root *yaml.Node, idx *index.SpecIndex) (T, error, bool, string) {
 	var circError error
-	if h, _, _ := utils.IsNodeRefValue(root); h {
+	var isReference bool
+	var referenceValue string
+	if h, _, rv := utils.IsNodeRefValue(root); h {
 		ref, err := LocateRefNode(root, idx)
 		if ref != nil {
 			root = ref
+			isReference = true
+			referenceValue = rv
 			if err != nil {
 				circError = err
 			}
 		} else {
 			if err != nil {
-				return nil, fmt.Errorf("object extraction failed: %s", err.Error())
+				return nil, fmt.Errorf("object extraction failed: %s", err.Error()), isReference, referenceValue
 			}
 		}
 	}
 	var n T = new(N)
 	err := BuildModel(root, n)
 	if err != nil {
-		return n, err
+		return n, err, isReference, referenceValue
 	}
 	err = n.Build(root, idx)
 	if err != nil {
-		return n, err
+		return n, err, isReference, referenceValue
 	}
 	// do we want to throw an error as well if circular error reporting is on?
 	if circError != nil && !idx.AllowCircularReferenceResolving() {
-		return n, circError
+		return n, circError, isReference, referenceValue
 	}
-	return n, nil
+	return n, nil, isReference, referenceValue
 }
 
 // ExtractObject will extract a typed Buildable[N] object from a root yaml.Node. The result is wrapped in a
@@ -147,11 +151,15 @@ func ExtractObjectRaw[T Buildable[N], N any](root *yaml.Node, idx *index.SpecInd
 func ExtractObject[T Buildable[N], N any](label string, root *yaml.Node, idx *index.SpecIndex) (NodeReference[T], error) {
 	var ln, vn *yaml.Node
 	var circError error
-	if rf, rl, _ := utils.IsNodeRefValue(root); rf {
+	var isReference bool
+	var referenceValue string
+	if rf, rl, refVal := utils.IsNodeRefValue(root); rf {
 		ref, err := LocateRefNode(root, idx)
 		if ref != nil {
 			vn = ref
 			ln = rl
+			isReference = true
+			referenceValue = refVal
 			if err != nil {
 				circError = err
 			}
@@ -163,10 +171,12 @@ func ExtractObject[T Buildable[N], N any](label string, root *yaml.Node, idx *in
 	} else {
 		_, ln, vn = utils.FindKeyNodeFull(label, root.Content)
 		if vn != nil {
-			if h, _, _ := utils.IsNodeRefValue(vn); h {
+			if h, _, rVal := utils.IsNodeRefValue(vn); h {
 				ref, lerr := LocateRefNode(vn, idx)
 				if ref != nil {
 					vn = ref
+					isReference = true
+					referenceValue = rVal
 					if lerr != nil {
 						circError = lerr
 					}
@@ -194,6 +204,8 @@ func ExtractObject[T Buildable[N], N any](label string, root *yaml.Node, idx *in
 		Value:       n,
 		KeyNode:     ln,
 		ValueNode:   vn,
+		IsReference: isReference,
+		Reference:   referenceValue,
 	}
 	// do we want to throw an error as well if circular error reporting is on?
 	if circError != nil && !idx.AllowCircularReferenceResolving() {
@@ -208,11 +220,15 @@ func ExtractArray[T Buildable[N], N any](label string, root *yaml.Node, idx *ind
 	*yaml.Node, *yaml.Node, error) {
 	var ln, vn *yaml.Node
 	var circError error
-	if rf, rl, _ := utils.IsNodeRefValue(root); rf {
+	var isReference bool
+	var referenceValue string
+	if rf, rl, rv := utils.IsNodeRefValue(root); rf {
 		ref, err := LocateRefNode(root, idx)
 		if ref != nil {
 			vn = ref
 			ln = rl
+			isReference = true
+			referenceValue = rv
 			if err != nil {
 				circError = err
 			}
@@ -223,10 +239,12 @@ func ExtractArray[T Buildable[N], N any](label string, root *yaml.Node, idx *ind
 	} else {
 		_, ln, vn = utils.FindKeyNodeFullTop(label, root.Content)
 		if vn != nil {
-			if h, _, _ := utils.IsNodeRefValue(vn); h {
+			if h, _, rv := utils.IsNodeRefValue(vn); h {
 				ref, err := LocateRefNode(vn, idx)
 				if ref != nil {
 					vn = ref
+					isReference = true
+					referenceValue = rv
 					if err != nil {
 						circError = err
 					}
@@ -271,6 +289,8 @@ func ExtractArray[T Buildable[N], N any](label string, root *yaml.Node, idx *ind
 			items = append(items, ValueReference[T]{
 				Value:       n,
 				ValueNode:   node,
+				IsReference: isReference,
+				Reference:   referenceValue,
 			})
 		}
 	}
@@ -325,11 +345,16 @@ func ExtractMapNoLookup[PT Buildable[N], N any](
 				currentKey = node
 				continue
 			}
+
+			var isReference bool
+			var referenceValue string
 			// if value is a reference, we have to look it up in the index!
-			if h, _, _ := utils.IsNodeRefValue(node); h {
+			if h, _, rv := utils.IsNodeRefValue(node); h {
 				ref, err := LocateRefNode(node, idx)
 				if ref != nil {
 					node = ref
+					isReference = true
+					referenceValue = rv
 					if err != nil {
 						circError = err
 					}
@@ -355,6 +380,8 @@ func ExtractMapNoLookup[PT Buildable[N], N any](
 			}] = ValueReference[PT]{
 				Value:       n,
 				ValueNode:   node,
+				IsReference: isReference,
+				Reference:   referenceValue,
 			}
 		}
 	}
@@ -378,15 +405,18 @@ func ExtractMap[PT Buildable[N], N any](
 	label string,
 	root *yaml.Node,
 	idx *index.SpecIndex) (map[KeyReference[string]]ValueReference[PT], *yaml.Node, *yaml.Node, error) {
-
+	var isReference bool
+	var referenceValue string
 	var labelNode, valueNode *yaml.Node
 	var circError error
-	if rf, rl, _ := utils.IsNodeRefValue(root); rf {
+	if rf, rl, rv := utils.IsNodeRefValue(root); rf {
 		// locate reference in index.
 		ref, err := LocateRefNode(root, idx)
 		if ref != nil {
 			valueNode = ref
 			labelNode = rl
+			isReference = true
+			referenceValue = rv
 			if err != nil {
 				circError = err
 			}
@@ -397,10 +427,12 @@ func ExtractMap[PT Buildable[N], N any](
 	} else {
 		_, labelNode, valueNode = utils.FindKeyNodeFull(label, root.Content)
 		if valueNode != nil {
-			if h, _, _ := utils.IsNodeRefValue(valueNode); h {
+			if h, _, rv := utils.IsNodeRefValue(valueNode); h {
 				ref, err := LocateRefNode(valueNode, idx)
 				if ref != nil {
 					valueNode = ref
+					isReference = true
+					referenceValue = rv
 					if err != nil {
 						circError = err
 					}
@@ -436,6 +468,8 @@ func ExtractMap[PT Buildable[N], N any](
 				v: ValueReference[PT]{
 					Value:       n,
 					ValueNode:   value,
+					IsReference: isReference,
+					Reference:   referenceValue,
 				},
 			}
 		}

datamodel/low/extraction_functions_test.go

@@ -520,12 +520,37 @@ func TestExtractObjectRaw(t *testing.T) {
 	var cNode yaml.Node
 	_ = yaml.Unmarshal([]byte(yml), &cNode)
 
-	tag, err := ExtractObjectRaw[*pizza](cNode.Content[0], idx)
+	tag, err, _, _ := ExtractObjectRaw[*pizza](cNode.Content[0], idx)
 	assert.NoError(t, err)
 	assert.NotNil(t, tag)
 	assert.Equal(t, "hello pizza", tag.Description.Value)
 }
 
+func TestExtractObjectRaw_With_Ref(t *testing.T) {
+
+	yml := `components:
+  schemas:
+    pizza:
+      description: hello`
+
+	var idxNode yaml.Node
+	mErr := yaml.Unmarshal([]byte(yml), &idxNode)
+	assert.NoError(t, mErr)
+	idx := index.NewSpecIndex(&idxNode)
+
+	yml = `$ref: '#/components/schemas/pizza'`
+
+	var cNode yaml.Node
+	_ = yaml.Unmarshal([]byte(yml), &cNode)
+
+	tag, err, isRef, rv := ExtractObjectRaw[*pizza](cNode.Content[0], idx)
+	assert.NoError(t, err)
+	assert.NotNil(t, tag)
+	assert.Equal(t, "hello", tag.Description.Value)
+	assert.True(t, isRef)
+	assert.Equal(t, "#/components/schemas/pizza", rv)
+}
+
 func TestExtractObjectRaw_Ref_Circular(t *testing.T) {
 
 	yml := `components:
@@ -548,7 +573,7 @@ func TestExtractObjectRaw_Ref_Circular(t *testing.T) {
 	var cNode yaml.Node
 	_ = yaml.Unmarshal([]byte(yml), &cNode)
 
-	tag, err := ExtractObjectRaw[*pizza](cNode.Content[0], idx)
+	tag, err, _, _ := ExtractObjectRaw[*pizza](cNode.Content[0], idx)
 	assert.Error(t, err)
 	assert.NotNil(t, tag)
 
@@ -570,7 +595,7 @@ func TestExtractObjectRaw_RefBroken(t *testing.T) {
 	var cNode yaml.Node
 	_ = yaml.Unmarshal([]byte(yml), &cNode)
 
-	tag, err := ExtractObjectRaw[*pizza](cNode.Content[0], idx)
+	tag, err, _, _ := ExtractObjectRaw[*pizza](cNode.Content[0], idx)
 	assert.Error(t, err)
 	assert.Nil(t, tag)
 
@@ -592,7 +617,7 @@ func TestExtractObjectRaw_Ref_NonBuildable(t *testing.T) {
 	var cNode yaml.Node
 	_ = yaml.Unmarshal([]byte(yml), &cNode)
 
-	_, err := ExtractObjectRaw[*test_noGood](cNode.Content[0], idx)
+	_, err, _, _ := ExtractObjectRaw[*test_noGood](cNode.Content[0], idx)
 	assert.Error(t, err)
 
 }
@@ -613,7 +638,7 @@ func TestExtractObjectRaw_Ref_AlmostBuildable(t *testing.T) {
 	var cNode yaml.Node
 	_ = yaml.Unmarshal([]byte(yml), &cNode)
 
-	_, err := ExtractObjectRaw[*test_almostGood](cNode.Content[0], idx)
+	_, err, _, _ := ExtractObjectRaw[*test_almostGood](cNode.Content[0], idx)
 	assert.Error(t, err)
 
 }

datamodel/low/reference.go

@@ -59,6 +59,12 @@ type NodeReference[T any] struct {
 
 	// The yaml.Node that is the key, that contains the value.
 	KeyNode *yaml.Node
+
+	// Is this value actually a reference in the original tree?
+	IsReference bool
+
+	// If HasReference is true, then Reference contains the original $ref value.
+	Reference string
 }
 
 // KeyReference is a low-level container for key nodes holding a Value of type T. A KeyNode is a pointer to the
@@ -81,6 +87,12 @@ type ValueReference[T any] struct {
 
 	// The yaml.Node that holds the referenced value
 	ValueNode *yaml.Node
+
+	// Is this value actually a reference in the original tree?
+	IsReference bool
+
+	// If HasReference is true, then Reference contains the original $ref value.
+	Reference string
 }
 
 // IsEmpty will return true if this reference has no key or value nodes assigned (it's been ignored)

datamodel/low/v2/definitions.go

@@ -84,12 +84,12 @@ func (d *Definitions) Build(root *yaml.Node, idx *index.SpecIndex) error {
 		var buildFunc = func(label *yaml.Node, value *yaml.Node, idx *index.SpecIndex,
 			r chan definitionResult[*base.SchemaProxy], e chan error) {
 
-			obj, err := low.ExtractObjectRaw[*base.SchemaProxy](value, idx)
+			obj, err, isRef, rv := low.ExtractObjectRaw[*base.SchemaProxy](value, idx)
 			if err != nil {
 				e <- err
 			}
 			r <- definitionResult[*base.SchemaProxy]{k: label, v: low.ValueReference[*base.SchemaProxy]{
-				Value: obj, ValueNode: value,
+				Value: obj, ValueNode: value, IsReference: isRef, Reference: rv,
 			}}
 		}
 		go buildFunc(defLabel, root.Content[i], idx, resultChan, errorChan)
@@ -144,11 +144,12 @@ func (pd *ParameterDefinitions) Build(root *yaml.Node, idx *index.SpecIndex) err
 		var buildFunc = func(label *yaml.Node, value *yaml.Node, idx *index.SpecIndex,
 			r chan definitionResult[*Parameter], e chan error) {
 
-			obj, err := low.ExtractObjectRaw[*Parameter](value, idx)
+			obj, err, isRef, rv := low.ExtractObjectRaw[*Parameter](value, idx)
 			if err != nil {
 				e <- err
 			}
-			r <- definitionResult[*Parameter]{k: label, v: low.ValueReference[*Parameter]{Value: obj, ValueNode: value}}
+			r <- definitionResult[*Parameter]{k: label, v: low.ValueReference[*Parameter]{Value: obj,
+				ValueNode: value, IsReference: isRef, Reference: rv}}
 		}
 		go buildFunc(defLabel, root.Content[i], idx, resultChan, errorChan)
 	}
@@ -192,11 +193,12 @@ func (r *ResponsesDefinitions) Build(root *yaml.Node, idx *index.SpecIndex) erro
 		var buildFunc = func(label *yaml.Node, value *yaml.Node, idx *index.SpecIndex,
 			r chan definitionResult[*Response], e chan error) {
 
-			obj, err := low.ExtractObjectRaw[*Response](value, idx)
+			obj, err, isRef, rv := low.ExtractObjectRaw[*Response](value, idx)
 			if err != nil {
 				e <- err
 			}
-			r <- definitionResult[*Response]{k: label, v: low.ValueReference[*Response]{Value: obj, ValueNode: value}}
+			r <- definitionResult[*Response]{k: label, v: low.ValueReference[*Response]{Value: obj,
+				ValueNode: value, IsReference: isRef, Reference: rv}}
 		}
 		go buildFunc(defLabel, root.Content[i], idx, resultChan, errorChan)
 	}
@@ -234,12 +236,12 @@ func (s *SecurityDefinitions) Build(root *yaml.Node, idx *index.SpecIndex) error
 		var buildFunc = func(label *yaml.Node, value *yaml.Node, idx *index.SpecIndex,
 			r chan definitionResult[*SecurityScheme], e chan error) {
 
-			obj, err := low.ExtractObjectRaw[*SecurityScheme](value, idx)
+			obj, err, isRef, rv := low.ExtractObjectRaw[*SecurityScheme](value, idx)
 			if err != nil {
 				e <- err
 			}
 			r <- definitionResult[*SecurityScheme]{k: label, v: low.ValueReference[*SecurityScheme]{
-				Value: obj, ValueNode: value,
+				Value: obj, ValueNode: value, IsReference: isRef, Reference: rv,
 			}}
 		}
 		go buildFunc(defLabel, root.Content[i], idx, resultChan, errorChan)

datamodel/low/v3/callback.go

@@ -48,7 +48,7 @@ func (cb *Callback) Build(root *yaml.Node, idx *index.SpecIndex) error {
 			currentCB = callbackNode
 			continue
 		}
-		callback, eErr := low.ExtractObjectRaw[*PathItem](callbackNode, idx)
+		callback, eErr, isRef, rv := low.ExtractObjectRaw[*PathItem](callbackNode, idx)
 		if eErr != nil {
 			return eErr
 		}
@@ -58,6 +58,8 @@ func (cb *Callback) Build(root *yaml.Node, idx *index.SpecIndex) error {
 		}] = low.ValueReference[*PathItem]{
 			Value:       callback,
 			ValueNode:   callbackNode,
+			IsReference: isRef,
+			Reference:   rv,
 		}
 	}
 	if len(callbacks) > 0 {