Added reference syntax.

README.md

@@ -178,3 +178,145 @@ using the following options.
 3. length, max_length: The length of the string - if not specified we
    use the terminator to find the end of the string. This can also be
    a lambda to derive the length from another field.
+
+
+### References
+
+When dereferencing a struct member we receive the basic type contained
+in the struct.
+
+Consider the following struct:
+
+```json
+[
+  ["TestStruct", 0, [
+     ["Field1", 2, "uint8"],
+     ["Field2", 0, Value, {
+         value: "x=>x.Field1 + 1"
+     }]
+  ]]
+]
+```
+
+In VQL, accessing the field will produce a uint8 type, so `Field2`
+will be calculated by adding 1 to the uint8 content of Field1.
+
+While this makes it easy and intuitive to use, sometimes we need to
+calculate some metadata over the struct field instead of it's literal
+value. For example metadata such as its starting offset, ending offset
+etc.
+
+Suppose we wanted to add another field, immediately following Field1:
+
+```json
+[
+  ["TestStruct", 0, [
+     ["Field1", 2, "uint8"],
+     ["Next", "x=>x.Field1.EndOf", "uint16"],
+  ]]
+]
+```
+
+This is not going to work, because `x.Field1` is a `uint8` integer
+type and it does not have an `EndOf` method.
+
+This is where references come in. We can obtain a struct field
+reference by using the @ character when accessing the field. A
+reference is a wrapper around the field which provides metadata about
+it.
+
+```json
+[
+  ["TestStruct", 0, [
+     ["Field1", 2, "uint8"],
+     ["Next", "x=>x.`@Field1`.RelEndOf", "uint16"],
+  ]]
+]
+```
+
+Note that the field name must be enclosed in backticks for VQL to
+identify the @ as part of the field name.
+
+Accessing a struct field with a name begining with @ will return a
+reference to the field instead of the field itself.
+
+The reference has many useful methods:
+
+- *SizeOf*, *Size*: These return the size of the field in bytes.
+- *StartOf*, *Start*, *OffsetOf*: These return the byte offset of the
+  beginning of the field.
+
+  NOTE that this offset is absolute - i.e. this offset will be
+  relative to the beginning of the file.
+
+- *RelOffset*: The relative offset of the field within the struct.
+
+  In the following, the Next and Next2 fields are equivalent:
+
+```json
+[
+  ["TestStruct", 0, [
+     ["Field1", 2, "uint8"],
+     ["Next", "x=>x.`@Field1`.StartOf + 4 - x.OffsetOf", "uint16"],
+     ["Next2", "x=>x.`@Field1`.RelOffset + 4", "uint16"],
+  ]]
+]
+```
+
+- *EndOf*, *End*: These return the end of the field in absolute bytes,
+  relative to the file.
+
+  In the following, the Next and Next2 fields are equivalent and are
+  both located directly after `Field1`:
+
+```json
+[
+  ["TestStruct", 0, [
+     ["Field1", 2, "uint8"],
+     ["Next", "x=>x.`@Field1`.EndOf - x.OffsetOf", "uint16"],
+     ["Next2", "x=>x.`@Field1`.RelOffset + x.`@Field1`.SizeOf", "uint16"],
+  ]]
+]
+```
+
+- *RelEndOf*: Like *EndOf* but relative to the start of the struct.
+
+  In particular notice that the profile struct syntax requires
+  specifying the offset of a field relative to the struct. Therefore
+  this field is especially useful to specify the field should be
+  located immediately after the previous field.
+
+```json
+[
+  ["TestStruct", 0, [
+     ["Field1", 2, "uint8"],
+     ["Next", "x=>x.`@Field1`.RelEndOf", "uint16"],
+  ]]
+]
+```
+
+  This also works for fields with variable size. For example consider
+  a null terminated string followed immediately with an integer.
+
+```json
+[
+  ["TestStruct", 0, [
+     ["Field1", 0, String],
+     ["Next", "x=>x.`@Field1`.RelEndOf", "uint32"],
+  ]]
+]
+```
+
+- *Value*: It is possible to dereference the reference to obtain the
+  real value.
+
+```json
+[
+  ["TestStruct", 0, [
+     ["Field1", 2, "uint8"],
+     ["Field2", 0, Value, {
+         value: "x=>x.`@Field1`.Value + 1"
+     }]
+  ]]
+]
+```

fixtures/TestStructParser.golden

@@ -1,20 +1,36 @@
 {
- "Field1": 3,
+ "Field1": 5,
  "Field2": {
-  "SecondField1": 7
+  "SecondField1": 9
  },
+ "StringField": "hello",
  "X": {
-  "Field1": 3,
+  "Field1": 5,
   "Field2": {
-   "SecondField1": 7
+   "SecondField1": 9
   },
-  "Field3": 578437695752307201,
+  "StringField": "hello",
+  "Field3": 1084818905618843912,
   "Field4": {
-   "SecondField1": 6
-  }
+   "SecondField1": 10
+  },
+  "OffsetOfField3": 7,
+  "SizeOfField3": 8,
+  "OffsetOfField2": 6,
+  "RelOffsetField2": 4,
+  "SizeOfField2": 5,
+  "StructOffset": 2,
+  "StringFieldSize": 12
  },
- "Field3": 578437695752307201,
+ "Field3": 1084818905618843912,
  "Field4": {
-  "SecondField1": 6
- }
+  "SecondField1": 10
+ },
+ "OffsetOfField3": 7,
+ "SizeOfField3": 8,
+ "OffsetOfField2": 6,
+ "RelOffsetField2": 4,
+ "SizeOfField2": 5,
+ "StructOffset": 2,
+ "StringFieldSize": 12
 }

parser.go

@@ -17,10 +17,17 @@ type Parser interface {
 	New(profile *Profile, options *ordereddict.Dict) (Parser, error)
 }
 
+// Used by parsers or wrappers who have fixed size
 type Sizer interface {
 	Size() int
 }
 
+// Applies on a parser which needs to instantiate to figure out the
+// size. i.e. the size depends on the data read (e.g. a string).
+type InstanceSizer interface {
+	InstanceSize(scope vfilter.Scope, reader io.ReaderAt, offset int64) int
+}
+
 // Allows psuedo elements to reveal their own value.
 type Valuer interface {
 	Value() interface{}

parser_test.go

@@ -44,6 +44,9 @@ var (
 
 		// offset 95 - E58E26 -> 624485
 		0xe5, 0x8e, 0x26,
+
+		// offset 96 - UTF16 string
+		0x68, 0x00, 0x65, 0x00, 0x6c, 0x00, 0x6c, 0x00, 0x6f, 0x00, 0x00, 0x00,
 	}
 )
 
@@ -87,9 +90,35 @@ func TestStructParser(t *testing.T) {
   ["TestStruct", "x => x.Field1 + 5", [
      ["Field1", 2, "uint8"],
      ["Field2", 4, "Second"],
+     ["StringField", 96, String, {
+         length: 20,
+         encoding: "utf16",
+     }],
      ["X", 0, "Value", {"value": "x=>x"}],
-     ["Field3", 0, "unsigned long long"],
-     ["Field4", "x => x.Field1", "Second"]
+     ["Field3", 5, "unsigned long long"],
+     ["Field4", "x => x.Field1", "Second"],
+     ["OffsetOfField3", 0, Value, {
+         ` + "value: 'x=>x.`@Field3`.OffsetOf', " + `
+     }],
+     ["SizeOfField3", 0, Value, {
+         ` + "value: 'x=>x.`@Field3`.SizeOf', " + `
+     }],
+     ["OffsetOfField2", 0, Value, {
+         ` + "value: 'x=>x.`@Field2`.OffsetOf', " + `
+     }],
+     ["RelOffsetField2", 0, Value, {
+         ` + "value: 'x=>x.`@Field2`.RelOffset', " + `
+     }],
+     ["SizeOfField2", 0, Value, {
+         ` + "value: 'x=>x.`@Field2`.SizeOf', " + `
+     }],
+     ["StructOffset", 0, Value, {
+        value: "x=>x.OffsetOf",
+     }],
+     ["StringFieldSize", 0, Value, {
+        ` + "value: 'x=>x.`@StringField`.SizeOf', " + `
+     }],
+
   ]],
 
   ["Second", 5, [
@@ -103,19 +132,19 @@ func TestStructParser(t *testing.T) {
 
 	// Parse TestStruct over the reader
 	reader := bytes.NewReader(sample)
-	obj, err := profile.Parse(scope, "TestStruct", reader, 0)
+	obj, err := profile.Parse(scope, "TestStruct", reader, 2)
 	assert.NoError(t, err)
 
 	// Field1 is at offset 2 has value 0x03
-	assert.Equal(t, uint64(3), Associative(scope, obj, "Field1"))
+	assert.Equal(t, uint64(5), Associative(scope, obj, "Field1"))
 
 	// Object size is calculated as x.Field1 + 5  ... 8
-	assert.Equal(t, 8, SizeOf(obj))
+	assert.Equal(t, 10, SizeOf(obj))
 
 	// Field4's offset is calculated as x=>x.Field1
 	// i.e. 3. SecondField1 has a relative offset of 2, therefore
 	// absolute offset of 3 + 2 = 5 -> value = 0x06
-	assert.Equal(t, uint64(6), Associative(scope, obj, "Field4.SecondField1"))
+	assert.Equal(t, uint64(10), Associative(scope, obj, "Field4.SecondField1"))
 
 	serialized, err := json.MarshalIndent(obj, "", " ")
 	assert.NoError(t, err)

protocol.go

@@ -34,7 +34,7 @@ func (self StructAssociative) Associative(scope vfilter.Scope,
 
 	// A Struct definition overrides default fields - this way a
 	// struct may define a field called "Offset" and it will be
-	// honored but if not defined we retur the default offset.
+	// honored but if not defined we return the default offset.
 	if lhs.HasField(rhs) {
 		return lhs.Get(rhs)
 	}
@@ -178,3 +178,65 @@ func (self ArrayIterator) Iterate(
 	return output_chan
 
 }
+
+type StructFieldReferenceAssociative struct{}
+
+func (self StructFieldReferenceAssociative) Applicable(a vfilter.Any, b vfilter.Any) bool {
+	switch a.(type) {
+	case StructFieldReference, *StructFieldReference:
+		_, ok := b.(string)
+		if ok {
+			return true
+		}
+	}
+	return false
+}
+
+func (self StructFieldReferenceAssociative) Associative(scope vfilter.Scope,
+	a vfilter.Any, b vfilter.Any) (vfilter.Any, bool) {
+	lhs, ok := a.(*StructFieldReference)
+	if !ok {
+		return vfilter.Null{}, false
+	}
+
+	rhs, ok := b.(string)
+	if !ok {
+		return vfilter.Null{}, false
+	}
+
+	switch rhs {
+	case "SizeOf", "Size":
+		return lhs.Size(), true
+
+	case "StartOf", "Start", "OffsetOf":
+		return lhs.Start(), true
+
+	case "RelOffset":
+		return lhs.RelOffset(), true
+
+	case "RelEndOf":
+		return lhs.RelOffset() + int64(lhs.Size()), true
+
+	case "EndOf", "End":
+		return lhs.End(), true
+
+	case "Value":
+		return lhs.Value(), true
+
+	default:
+		return nil, false
+	}
+}
+
+func (self StructFieldReferenceAssociative) GetMembers(scope vfilter.Scope, a vfilter.Any) []string {
+	return nil
+}
+
+func GetProtocols() []vfilter.Any {
+	return []vfilter.Any{
+		&StructAssociative{},
+		&ArrayAssociative{},
+		&ArrayIterator{},
+		&StructFieldReferenceAssociative{},
+	}
+}