From ff83de22a4715aedea96158cf96ea106032b44c4 Mon Sep 17 00:00:00 2001 From: David Van Horn Date: Tue, 21 Oct 2025 10:56:58 -0400 Subject: [PATCH 1/4] Rudimentary utility for making heap diagrams. --- www/notes/diagrams.rkt | 144 +++++++++++++++++++++++++++++++++++++++++ 1 file changed, 144 insertions(+) create mode 100644 www/notes/diagrams.rkt diff --git a/www/notes/diagrams.rkt b/www/notes/diagrams.rkt new file mode 100644 index 00000000..2f29f9bc --- /dev/null +++ b/www/notes/diagrams.rkt @@ -0,0 +1,144 @@ +#lang racket +(provide make-heap-diagram) +(require pict) +(require pict/code) + +(define pi 3.14) + + +(define n 40) + +(define (make-imm-cell i) + (cc-superimpose + (code #,i) + (rectangle n n))) + +(define (make-cons-cell) + (cb-superimpose (rectangle n n) + (code cons))) + + +(define (make-box-cell) + (cb-superimpose (rectangle n n) + (code box))) + + +(define (fwd-pts-to a b p) + (pin-arrow-line 7 p + a cc-find + b lt-find + #:start-angle (/ pi 2) + #:end-angle (- (/ pi 2)) + #:start-pull 1/4 + #:end-pull 1/2)) + + +#| +(define rax (make-cons-cell)) +(define m + (let ((a (make-imm-cell 1)) + (b (make-cons-cell)) + (c (make-imm-cell 2)) + (d (make-cons-cell)) + (e (make-imm-cell 3)) + (f (make-imm-cell ''()))) + (define pre + (foldr (λ (p1 p2) + (hc-append 0 p1 p2)) + (rectangle 0 n) + (list a b c d e f))) + (define heap + (vc-append 0 (fwd-pts-to d e (fwd-pts-to b c pre)) + (text "heap"))) + + (define all + (hc-append n (vc-append 0 rax (text "rax")) heap)) + + (define q + (fwd-pts-to rax heap all)) + + (inset q 20))) +|# + + + +(define (make-cell v) + (match v + [`(cons ,_) (make-cons-cell)] + [`(box ,_) (make-box-cell)] + [_ (make-imm-cell v)])) + +(define (add-arrows spec cells p) + ;(printf "~a~n" spec) + (match spec + ['() p] + [(cons `(cons ,i) s) + (add-arrows s + cells + (fwd-pts-to (list-ref cells (sub1 (- (length cells) (length s)))) + (list-ref cells i) + p))] + [(cons _ s) (add-arrows s cells p)])) + +(define (make-heap-diagram spec) + (match spec + [(cons (and `(,_ ,i) r) h) + (define rax (make-cell r)) + (define heap (map make-cell h)) + (define heap/arrows + (add-arrows (rest spec) heap + (foldr (λ (p1 p2) + (hc-append 0 p1 p2)) + (rectangle 0 n) + heap))) + + (define heap/arrows/label + (vc-append + 0 + heap/arrows + (text "heap"))) + + (define rax/label + (vc-append 0 rax (text "rax"))) + + (inset + (fwd-pts-to rax (list-ref heap i) (hc-append n rax/label heap/arrows/label)) + (* n 2))])) +#; +(make-heap-diagram + '((cons 0) + 1 + (cons 2) + 2 + (cons 4) + 3 + '())) + +#; +(make-heap-diagram + '((cons 4) + 3 + '() + 2 + (cons 0) + 1 + (cons 2))) + + + + +#; +(let ((a (make-imm-cell 3)) + (b (make-imm-cell ''())) + (c (make-imm-cell 2)) + (d (make-cons-cell)) + (e (make-imm-cell 1)) + (f (make-cons-cell)) + (g (make-cocell ''()))) + (define pre + (foldr (λ (p1 p2) + (hc-append 0 p1 p2)) + (rectangle 0 n) + (list a b c d e f g))) + (inset (fwd-pts-to f g (fwd-pts-to d e (fwd-pts-to b c pre))) 20)) + From 5280a7647e313ee535abc3ba06453c72521b7e7a Mon Sep 17 00:00:00 2001 From: David Van Horn Date: Tue, 21 Oct 2025 11:08:28 -0400 Subject: [PATCH 2/4] Add hex utility for writing hex literals in notes. --- www/notes/utils.rkt | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/www/notes/utils.rkt b/www/notes/utils.rkt index 552be1f9..444e3d68 100644 --- a/www/notes/utils.rkt +++ b/www/notes/utils.rkt @@ -36,14 +36,20 @@ (define (save-file f s) (with-output-to-file f (λ () (display s)) #:exists 'replace)) -(define (binary i [len 0]) +(define (base i b [len 0]) (typeset-code #:block? #f #:indent 0 - (string-append "#b" - (~a (number->string i 2) + (string-append "#" + (case b [(2) "b"] [(16) "x"]) + (~a (string-upcase (number->string i b)) #:left-pad-string "0" #:align 'right #:min-width len)))) +(define (hex i [len 0]) + (base i 16 len)) +(define (binary i [len 0]) + (base i 2 len)) + (define (src-code lang) (margin-note (small-save-icon) " " (link (string-append "code/" (string-downcase lang) ".zip") "Source code") From 9140e397020df5e9a47a75589a77a639483c9022 Mon Sep 17 00:00:00 2001 From: David Van Horn Date: Thu, 23 Oct 2025 21:28:34 -0400 Subject: [PATCH 3/4] Add vect cells to diagrams. --- www/notes/diagrams.rkt | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/www/notes/diagrams.rkt b/www/notes/diagrams.rkt index 2f29f9bc..c4928cdb 100644 --- a/www/notes/diagrams.rkt +++ b/www/notes/diagrams.rkt @@ -22,6 +22,11 @@ (cb-superimpose (rectangle n n) (code box))) +(define (make-vect-cell) + (cb-superimpose (rectangle n n) + (code vect))) + + (define (fwd-pts-to a b p) (pin-arrow-line 7 p @@ -66,13 +71,14 @@ (match v [`(cons ,_) (make-cons-cell)] [`(box ,_) (make-box-cell)] + [`(vect ,_) (make-vect-cell)] [_ (make-imm-cell v)])) (define (add-arrows spec cells p) ;(printf "~a~n" spec) (match spec ['() p] - [(cons `(cons ,i) s) + [(cons `(_ ,i) s) (add-arrows s cells (fwd-pts-to (list-ref cells (sub1 (- (length cells) (length s)))) From bc2fd2044241cd1f0d40c81943a9d9e84d7f6f88 Mon Sep 17 00:00:00 2001 From: David Van Horn Date: Thu, 23 Oct 2025 21:29:35 -0400 Subject: [PATCH 4/4] Overhaul Hustle and Hoax and fix up based on langs changes. --- www/notes/evildoer.scrbl | 12 +- www/notes/fraud.scrbl | 8 +- www/notes/hoax.scrbl | 1067 +++++++++++++++++++++++++++----- www/notes/hustle.scrbl | 1253 ++++++++++++++++++++++++++++---------- www/notes/knock.scrbl | 4 +- www/notes/loot.scrbl | 52 +- 6 files changed, 1880 insertions(+), 516 deletions(-) diff --git a/www/notes/evildoer.scrbl b/www/notes/evildoer.scrbl index 794bc823..d7cfb960 100644 --- a/www/notes/evildoer.scrbl +++ b/www/notes/evildoer.scrbl @@ -702,16 +702,12 @@ of @racket[interp/io]: @ex[ (exec/io (parse '(write-byte (read-byte))) "z")] -Note that we still provide an @racket[exec] function that works for -programs that don't do I/O: +Note that we still provide an @racket[exec] function, but it +assumes there is no input and it prints all output: @ex[ -(exec (parse '(eof-object? #f)))] - -But it will fail if executing a program that uses I/O: - -@ex[ -(eval:error (exec (parse '(write-byte 97))))] +(exec (parse '(eof-object? (read-byte)))) +(exec (parse '(write-byte 97)))] We can now state the correctness property we want of the compiler: diff --git a/www/notes/fraud.scrbl b/www/notes/fraud.scrbl index 1120f3fb..d21d166a 100644 --- a/www/notes/fraud.scrbl +++ b/www/notes/fraud.scrbl @@ -899,10 +899,10 @@ domain of expressions to be just @bold{closed expressions}, i.e. those that have no unbound variables. @bold{Compiler Correctness}: @emph{For all @racket[e] @math{∈} -@tt{ClosedExpr}, @racket[i], @racket[o] @math{∈} @tt{String}, and @racket[v] -@math{∈} @tt{Value}, if @racket[(interp/io e i)] equals @racket[(cons -v o)], then @racket[(exec/io e i)] equals -@racket[(cons v o)].} +@tt{ClosedExpr}, @racket[i], @racket[o] @math{∈} @tt{String}, and @racket[A] +@math{∈} @tt{Answer}, if @racket[(interp/io e i)] equals @racket[(cons +a o)], then @racket[(exec/io e i)] equals +@racket[(cons a o)].} The check for correctness is the same as before, although the check should only be applied to elements of @tt{ClosedExpr}: diff --git a/www/notes/hoax.scrbl b/www/notes/hoax.scrbl index 2a606f9d..c65bb4e2 100644 --- a/www/notes/hoax.scrbl +++ b/www/notes/hoax.scrbl @@ -4,21 +4,19 @@ @(require redex/pict racket/runtime-path scribble/examples + hoax/types ; (except-in "../../langs/hustle/semantics.rkt" ext lookup) ; (prefix-in sem: (only-in "../../langs/hustle/semantics.rkt" ext lookup)) "../fancyverb.rkt" "utils.rkt" "ev.rkt" + "diagrams.rkt" "../utils.rkt") @(define codeblock-include (make-codeblock-include #'h)) -@(ev '(require rackunit a86)) -@(ev `(current-directory ,(path->string (build-path langs "hoax")))) -@(void (ev '(with-output-to-string (thunk (system "make runtime.o"))))) -@(for-each (λ (f) (ev `(require (file ,f)))) - '("interp.rkt" "compile.rkt" "compile-ops.rkt" "ast.rkt" "parse.rkt" "types.rkt")) - +@(ev '(require rackunit a86 hoax hoax/compile-ops hoax/assert)) + @(define this-lang "Hoax") @title[#:tag this-lang]{@|this-lang|: vectors and strings} @@ -129,29 +127,101 @@ Now that were are comfortable with heap-allocated data-structures like boxes and pairs, handling vectors is not too difficult. Vectors are similarly heap allocated. This will require a new kind of pointer value: -@verbatim{ -- values - + pointers (non-zero in last 3 bits) - * boxes - * pairs - * vectors - * strings - + immediates (zero in last three bits) - * integers - * characters - * booleans - * ... -} +@itemlist[ +@item{Values + @itemlist[@item{pointers (non-zero in last 3 bits) + @itemlist[@item{Boxes} @item{Pairs} @item{Vectors} @item{Strings}]} + @item{immediates (zero in last three bits) + @itemlist[@item{integers} @item{characters} @item{booleans} @item{...}]}]}] We will follow exactly the same scheme we followed for box and pairs: vectors and strings will be uniquely tagged in the lowest three bits and the remaining bits will indicate an address in memory which can be obtained by zeroing out the tag bits. + +@;{ + +@section{Detour: static data} + +Before looking at how we can implement the vector and string +operations, let's take a quick side quest to talk about @bold{static +memory}. It will be useful in making examples and it's going to help +with a small issue concerning the empty vector and empty string. It +will also be useful in the future when we get to adding compound +literals to our language. + +Static data is memory that is allocated before a program is run. The +mechanism for allocating this data is to actually use space @emph{in} +the program to hold the data. The program after all, like everything, +is just a blob of bits. When it's executed, the operating system +loads the program from a file into memory. We can use parts of that +memory at run-time if we'd like. Typically that memory contains +executable instructions, but also we make space in the program to hold +arbitrary data. The way to do this is to create a @bold{data section} +by using the @racket[(Data)] psuedo-instruction. It's not actually an +instruction, rather it instructs the assembler that the subsequent +content of the program should be part of the data section in the +object file. Within the @racket[Data] section, we can use the +@racket[Dq] ``instruction'' for defining an 8-byte quantity that is +placed directly in the file. + +Here's an example where we make a @emph{static} pair. Notice that the +program does not allocate any memory at run-time (no @tt{malloc} or +@racket[rbx] register used), yet it manages to return a pair to +the run-time system: + +@ex[ +(run + (prog + (Global 'entry) + (Label 'entry) + (Lea rax (Mem 'p type-cons)) + (Ret) + (Data) + (Label 'p) + (Dq (value->bits #t)) + (Dq (value->bits #f))))] + +It does this by embedding the pair itself into the text of the program +and giving the location of this data a name with the label +@racket['p]. When it's executed, we construct a tagged pointer into +the data section and return that. + +From the run-time's perspective there's no difference between this and +pair that's allocated in the heap. It's just a tagged pointer to two +words of memory. + +We're able to use this trick because we knew ahead of time that we +want to represent @racket['(1 . 2)]. In general we can't know ahead +of time what memory a program needs to allocate or what data will be +stored in that memory, so we still need dynamically allocated memory, +but this is a useful mechanism to know about. + +We can of course represent other kinds of data using this trick. +Here's a static vector @racket['#(1 2 3)]: + +@ex[ +(run + (prog + (Global 'entry) + (Label 'entry) + (Lea rax (Mem 'v type-vect)) + (Ret) + (Data) + (Label 'v) + (Dq (value->bits 3)) + (Dq (value->bits 1)) + (Dq (value->bits 2)) + (Dq (value->bits 3))))] + +} + @section{Representing and operating on vectors} -The memory that is pointed to by a vector pointer will contain the -size of the vector followed by that many words of memory, one for each +Vectors are sized @bold{heterogenous arrays} of values. The memory +that is pointed to by a vector tagged pointer will contain the length +of the vector followed by that many words of memory, one for each element of the vector. (Strings will be similar, with a slight twist, which we'll examine later.) @@ -159,105 +229,610 @@ So for example the following creates a vector of size 3 containing the values @racket[1], @racket[#t], @racket[#\c]: @#reader scribble/comment-reader -(racketblock -(seq (Mov (Offset 'rbx 0) 3) ; write vector length 3 - (Mov 'rax (value->bits 1)) - (Mov (Offset 'rbx 8) 'rax) ; write 1 in vector slot 0 - (Mov 'rax (value->bits #t)) - (Mov (Offset 'rbx 16) 'rax) ; write #t in vector slot 1 - (Mov 'rax (value->bits #\c)) - (Mov (Offset 'rbx 24) 'rax) ; write #\c in vector slot 2 - (Mov 'rax 'rbx) - (Or 'rax type-vect) ; tag pointer as a vector - (Add 'rbx 32)) ; advance rbx four words -) +(ex + (run + (prog + (Global 'entry) + (Label 'entry) + ;; Set up heap pointer from run-time system + (Push rbx) + (Mov rbx rdi) + ;; Create vector + (Mov rax (value->bits 3)) + (Mov (Mem rbx 0) rax) ; write vector length 3 + (Mov rax (value->bits 1)) + (Mov (Mem rbx 8) rax) ; write value 1 + (Mov rax (value->bits #t)) + (Mov (Mem rbx 16) rax) ; write value #t + (Mov rax (value->bits #\c)) + (Mov (Mem rbx 24) rax) ; write value #\c + (Mov rax rbx) + (Xor rax type-vect) ; tag pointer as vector + (Add rbx 32) ; acct for memory used + ;; Restore registers and return + (Pop rbx) + (Ret)))) + +This creates a vector in memory like this: + +@make-heap-diagram['((vect 0) 3 1 #t #\c)] + +In order to explore things interactively, let's set up a little +function for running code in the context of this vector. It will take +given instructions and run them after @racket['#(1 #t #\c)] +is in @racket[rax]: + + +@#reader scribble/comment-reader +(ex + (define (eg is) + (run + (prog + (Global 'entry) + (Extern 'raise_error) + (Label 'entry) + ;; Set up heap pointer from run-time system + (Push rbx) + (Mov rbx rdi) + ;; Create vector + (Mov rax (value->bits 3)) + (Mov (Mem rbx 0) rax) ; write vector length 3 + (Mov rax (value->bits 1)) + (Mov (Mem rbx 8) rax) ; write value 1 + (Mov rax (value->bits #t)) + (Mov (Mem rbx 16) rax) ; write value #t + (Mov rax (value->bits #\c)) + (Mov (Mem rbx 24) rax) ; write value #\c + (Mov rax rbx) + (Xor rax type-vect) ; tag pointer as vector + (Add rbx 32) ; acct for memory used + + ;; include given instructions + is + + ;; Restore registers and return + (Pop rbx) + (Ret) + (Label 'err) + (Call 'raise_error))))) + +If we call @racket[eg] with no instructions, we just get the vector +itself back: + +@ex[(eg (seq))] + +OK, now let's think about some of the operations we want to implement, +starting with @racket[vector-length]. Since the vector stores its +length as the first word in memory, implementing +@racket[vector-length] amounts to fetching that word of memory: + +@ex[(eg (seq (Mov rax (Mem rax (- 0 type-vect)))))] + +The operation should also do some type tag checking to make sure its +argument actually is a vector first. We can define a helper function +for asserting the ``vectorness'' of something in a given register: + +@ex[(assert-vector rax)] + +Of course for our example, the assertion succeeds and we still get its +length: -Notice that the value written at offset @racket[0] is @racket[3], not -@racket[(value->bits 3)]. This is because this slot of memory in a -vector can only hold an integer, not an arbitrary value, so there's no -need to encode the type into the value---it's position tells us it's -an integer. +@ex[ + (eg (seq (assert-vector rax) + (Mov rax (Mem rax (- 0 type-vect)))))] + + +This will be our implementation of @racket[vector-length], now let's +do @racket[vector-ref]. If @racket[rax] contains a vector, we can +access the element at index @racket[_i] by dereferencing the memory at +location @racket[(- (add1 (* _i 8)) type-vect)]. The @racket[add1] is +needed to account for the word at the start that holds the length. So +for example, to get the three elements of our example vector: + +@ex[ +(eg (seq (Mov rax (Mem rax (- 8 type-vect))))) +(eg (seq (Mov rax (Mem rax (- 16 type-vect))))) +(eg (seq (Mov rax (Mem rax (- 24 type-vect)))))] + +This is the essence of @racket[vector-ref], but there's a bit more to +the story. First, we don't actually know what index were fetching +until run-time. Because @racket[vector-ref] is a binary operation and +the order of the arguments is the vector, then the index, we will have +the vector value on the stack and index value in @racket[rax]. We can +simulate this in our example by pushing the vector and writing our +index into @racket[rax]. We then need instructions to (1) pop the +stack into a temporary register and (2) deference the memory using the +@racket[rax] instead of a literal offset: -Now let's consider referencing elements of a vector. If @racket['rax] -holds a vector value, we can reference an element of the vector by -untagging the value and fetching from an appropriate offset. Suppose -we want to fetch the 2nd element (i.e. index @racket[1]) of a vector -in @racket['rax]: +@#reader scribble/comment-reader +(ex + (eg (seq (Push rax) + (Mov rax 0) ; index = 0 + ;; Start of vector-ref code + (Pop r8) + (Mov rax (Mem r8 rax (- 8 type-vect)))))) + +This example is a little deceiving because the index we happened to +want is @racket[0], but remember that the argument to +@racket[vector-ref] will be a value, so we really should have set +@racket[rax] to @racket[(value->bits 0)]. We got lucky here because +the value @racket[0] is represented by the bits @racket[0]. But let's +think about an index of @racket[1]. If @racket[rax] holds the +@emph{value} @racket[1], then that will be the bits +@racket[#,(value->bits 1)]. If @racket[rax] holds the value +@racket[2], then that will be the bits @racket[#,(value->bits 2)]. +But those aren't the right offsets that we want; we want offsets +@racket[8] and @racket[16] respectively. In general, and index value +@racket[_i] will be represented by the bits @racket[(* _i 16)], but +want offset @racket[(* _i 8)]. The simplest way to convert from the +value @racket[_i] to the appropriate offset is to shift to the right +1 bit. So here's how can access all the elements uniformly, starting +from an index value in @racket[rax]: @#reader scribble/comment-reader -(racketblock -(seq (Xor 'rax type-vect) ; erase the vector tag - (Mov 'rax (Offset 'rax 16))) ; load index 1 into rax -) +(ex + (eg (seq (Push rax) + (Mov rax (value->bits 0)) ; index = 0 + ;; Start of vector-ref code + (Pop r8) + (Sar rax 1) ; convert int to byte offset + (Mov rax (Mem r8 rax (- 8 type-vect))))) + (eg (seq (Push rax) + (Mov rax (value->bits 1)) ; index = 1 + ;; Start of vector-ref code + (Pop r8) + (Sar rax 1) ; convert int to byte offset + (Mov rax (Mem r8 rax (- 8 type-vect))))) + (eg (seq (Push rax) + (Mov rax (value->bits 2)) ; index = 2 + ;; Start of vector-ref code + (Pop r8) + (Sar rax 1) ; convert int to byte offset + (Mov rax (Mem r8 rax (- 8 type-vect)))))) + +We're getting closer. We now just need to do a little more error checking. +First, we should check that @racket[rax] holds a non-negative integer +and that the stack argument is a vector. Here we have a little helper +for asserting that something is a natural number: + +@ex[(assert-natural rax)] + +So we can add in the type checking like this: -Notice that the offset here is @racket[16] because the first word is -the length, so the second word is the first element, and the third -word (offset 16) is the element we want. +@#reader scribble/comment-reader +(ex + (eg (seq (Push rax) + (Mov rax (value->bits 2)) ; index = 2 + ;; Start of vector-ref code + (Pop r8) + (assert-natural rax) + (assert-vector r8) + (Sar rax 1) ; convert int to byte offset + (Mov rax (Mem r8 rax (- 8 type-vect)))))) + +Finally, we have just a little more checking to do. We need to be +sure to signal an error if you go off the end of a vector, which our +code currently does not do: -This code assumes the vector has a length of at least two. In -general, the vector operations must check that the given index is -valid for the vector. This is accomplished by checking against the -length stored in the first word of the vector's memory. Using -@racket['r9] as a scratch register, we could insert a check as -follows: +@#reader scribble/comment-reader +(ex + (eg (seq (Push rax) + (Mov rax (value->bits 7)) ; index = 7 + ;; Start of vector-ref code + (Pop r8) + (assert-natural rax) + (assert-vector r8) + (Sar rax 1) ; convert int to byte offset + (Mov rax (Mem r8 rax (- 8 type-vect)))))) + +Here we read past the end of the vector by using an index of +@racket[7] on a vector of length @racket[3]. That should be an error, +otherwise it's possible to access arbitrary memory. To do this, we +need to fecth the length and make sure the index value is less than +the length value. Now when we use an index of @racket[7], we get an +error, but for a valid index, we get the appropriate element: @#reader scribble/comment-reader -(racketblock -(seq (Xor 'rax type-vect) ; erase the vector tag - (Mov 'r9 (Offset 'rax 0)) ; load length into r9 - (Cmp 'r9 2) ; see if len < 2, - (Jl 'raise_error) ; raise error if so, otherwise - (Mov 'rax (Offset 'rax 16))) ; load index 1 into rax -) +(ex + (eg (seq (Push rax) + (Mov rax (value->bits 7)) ; index = 7 + ;; Start of vector-ref code + (Pop r8) + (assert-natural rax) + (assert-vector r8) + (Mov r9 (Mem r8 (- type-vect))) + (Cmp rax r9) + (Jge 'err) + (Sar rax 1) + (Mov rax (Mem r8 rax (- 8 type-vect))))) + (eg (seq (Push rax) + (Mov rax (value->bits 2)) ; index = 2 + ;; Start of vector-ref code + (Pop r8) + (assert-natural rax) + (assert-vector r8) + (Mov r9 (Mem r8 (- type-vect))) + (Cmp rax r9) + (Jge 'err) + (Sar rax 1) + (Mov rax (Mem r8 rax (- 8 type-vect)))))) + +We've got @racket[vector-ref] down. Let's turn to +@racket[vector-set!]. At its core, we're just doing a memory write to +replace a vector element with a new one. For example, here's how we +can update the second element to be @racket[#f]: -Suppose @racket['rax] holds a vector value and we want to update the -2nd element (i.e. index @racket[1]) to be @racket[#f]. Following the -outline above, we can erase the vector tag, check that the index is -valid, and then, rather than loading the element from memory, we can -write the new element at the appropriate offset: +@ex[ + (eg (seq (Mov r8 (value->bits #f)) + (Mov (Mem rax (- 16 type-vect)) r8)))] + +But of course there's more to it than this. Just like +@racket[vector-ref] we need to do some type- and bounds-checking. We +also have to deal with the fact that this is a ternary primitive---it +takes 3 arguments. The first two will be on the stack and the third +will be in @racket[rax]. The first argument is the vector, the second +is in the index, and the third is the element to write into the +vector. We can get pretty close with this: @#reader scribble/comment-reader -(racketblock -(seq (Xor 'rax type-vect) ; erase the vector tag - (Mov 'r9 (Offset 'rax 0)) ; load length into r9 - (Cmp 'r9 2) ; see if len < 2, - (Jl 'raise_error) ; raise error if so, otherwise - (Mov (Offset 'rax 16) - (value->bits #f))) ; ; write #f into index 1 -) +(ex + (eg (seq (Push rax) + (Mov rax (value->bits 2)) ; index = 2 + (Push rax) + (Mov rax (value->bits #f)) + ;; Start of vector-set! code + (Mov r10 rax) ; update value + (Pop rax) ; index + (Pop r8) ; vector + (assert-natural rax) + (assert-vector r8) + (Mov r9 (Mem r8 (- type-vect))) + (Cmp rax r9) + (Jge 'err) + (Sar rax 1) + (Mov (Mem r8 rax (- 8 type-vect)) r10)))) + +Although it's impossible to verify from the result, this actually +@emph{has} the intended effect of modifying the vector. It's been +written to mimic the code for @racket[vector-ref] by placing the +vector in @racket[r8] and the index in @racket[rax]. The value +that's going to be written gets moved over to another temporary +register, @racket[r10]. The type- and bounds-checking is exactly as +in @racket[vector-ref]. The only change is that the last instruction +is a write instead of a read. + +But why did we get @racket[1] in the end? Well that's just what +happens to be in @racket[rax]. We had the index value there, then we +shifted to the right to convert it to a byte offset. That happens be +interpreted as the value @racket[1]. What value should we have ended +up with? The answer that Racket goes with is @racket[(void)], which +signifies that the expression is evaluated for effect. That's easy +enough, we can just move @racket[(void)] into @racket[rax] at the very +end: + +@#reader scribble/comment-reader +(ex + (eg (seq (Push rax) + (Mov rax (value->bits 2)) ; index = 2 + (Push rax) + (Mov rax (value->bits #f)) + ;; Start of vector-set! code + (Mov r10 rax) ; update value + (Pop rax) ; index + (Pop r8) ; vector + (assert-natural rax) + (assert-vector r8) + (Mov r9 (Mem r8 (- type-vect))) + (Cmp rax r9) + (Jge 'err) + (Sar rax 1) + (Mov (Mem r8 rax (- 8 type-vect)) r10) + (Mov rax (value->bits (void)))))) + +Notice that it appears nothing is returned, but this is just the +REPL's handling of @racket[(void)]: it's printed form is empty. + +This is the correct code for @racket[vector-set!], but how do we see +that it did the right thing to the vector? How do we confirm that it +had the intended effect? Well in this example, we lost our handle on +the vector. We can adapt the example to save a way a copy of the +vector and then restore it at the end in order to observe the change +happened. Here we do that by saving the vector on the stack (twice: +once to restore later and once as an argument to +@racket[vector-set!]): + +@#reader scribble/comment-reader +(ex + (eg (seq (Push rax) ; save vector + (Push rax) + (Mov rax (value->bits 2)) ; index = 2 + (Push rax) + (Mov rax (value->bits #f)) + ;; Start of vector-set! code + (Mov r10 rax) ; update value + (Pop rax) ; index + (Pop r8) ; vector + (assert-natural rax) + (assert-vector r8) + (Mov r9 (Mem r8 (- type-vect))) + (Cmp rax r9) + (Jge 'err) + (Sar rax 1) + (Mov (Mem r8 rax (- 8 type-vect)) r10) + (Mov rax (value->bits (void))) + ;; End of vector-set! code + ;; Restore to see modification + (Pop rax)))) + + +As you can see, the element at index @racket[2] has changed to +@racket[#f]. + +So we've now done @racket[vector-length], @racket[vector-ref], +@racket[vector-set!]. What about @racket[make-vector]? + +The @racket[make-vector] operation takes two arguments: the length of +the vector to construct and a value used to initialize each element of +the vector. Note that neither of these values are known at +compile-time, in general. + +Now if we punt on initialization, making a vector is pretty easy: + +@#reader scribble/comment-reader +(ex + (eg (seq (Mov rax (value->bits 3)) ; length argument + (Push rax) + (Mov rax (value->bits #t)) ; init argument + ;; Start of make-vector code + (Pop r8) + (assert-natural r8) + (Mov (Mem rbx 0) r8) ; write length + (Mov rax rbx) + (Xor rax type-vect) ; create tagged pointer + (Add rbx 8) ; acct for stored length + (Sar r8 1) ; convert to bytes, acct for elements + (Add rbx r8))) + ) + +This code checks the type of the length argument to make sure its +valid, then writes the length to memory, constructs a tagged pointer +to that memory, and then adjusts the heap pointer based on the length +of the vector. But you'll notice the vector shows up as containing +@racket[0]s. That's just an artifact of our heap being zeroed out to +start, which is not guaranteed and we could have any possible bits for +elements in the vector, including bits that do not encode any value, +or worse, bits that appear to encode tagged pointer value, but point +to invalid memory locations. So we really need to initialize the +elements to avoid this undefined and dangerous behavior. + +To do that, we need to generate a run-time @emph{loop}. The loop +needs to cycle through as many times as the length argument, writing +the initialization value each time to the appropriate location in +memory. We can't avoid a loop because we can't (in general) know what +the length argument is going to be until run-time. + + +@#reader scribble/comment-reader +(ex + (eg (seq (Mov rax (value->bits 3)) ; length argument + (Push rax) + (Mov rax (value->bits #t)) ; init argument + ;; Start of make-vector code + (Pop r8) + (assert-natural r8) + + (Mov (Mem rbx 0) r8) ; write length + (Sar r8 1) ; convert to bytes + (Mov r9 r8) ; save for heap adjustment + + ;; start initialization + (Label 'loop) + (Cmp r8 0) + (Je 'done) + (Mov (Mem rbx r8) rax) + (Sub r8 8) + (Jmp 'loop) + (Label 'done) + ;; end initialization + + (Mov rax rbx) + (Xor rax type-vect) ; create tagged pointer + (Add rbx r9) ; acct for elements and stored length + (Add rbx 8)))) + + +Now this code successfully does the initialization and everything +seems good... but there is a corner case where this code could be +improved. Consider the case of @racket[(make-vector 0 #t)]. This +code will dutiful create an empty vector, represented in memory like +this: + +@make-heap-diagram['((vect 0) 0)] + +It uses up one word of memory to store the length, which is +@racket[0]. That doesn't seem so bad, but consider a program like + +@racketblock[ +(begin (make-vector 0 #t) + (make-vector 0 #f))] + + +This will create @emph{two} empty vectors, using up two words of +memory: + +@make-heap-diagram['((vect 1) 0 0)] + +Of course, both of the empty vectors are identical; every empty vector +is identical: its a tagged pointer to a single word containing +@racket[0]. It seems silly to use memory at all since we kind of know +everything there is to know about the empty vector already. Moreover, +we usually think of (and talk about) @emph{the} empty vector, rather +than @emph{an} empty vector, because really having just a single empty +vector is all you need. + +How can we take this idea and incorporate it into our design. First, +let's see how we can make a single, canonical representation of the +empty vector. We will do this by using a @bold{data section} to +@bold{statically allocate} memory for representing our canonical empty +vector. + +@ex[ +(eg (seq (Data) + (Label 'empty) + (Dq 0) + (Text) + (Lea rax (Mem 'empty type-vect))))] + + +This is our first encounter with a data section, so let's dig in a +little. The @racket[Data] declaration tells the assembler that +instructions that follow should live in the ``data'' part of the +program, rather than the ``text'' part that contains executable +instructions. Within the data section, we use the @racket[Dq] +``psuedo-instruction'' to specify exactly the bits we want placed in +the object file; @racket[Dq] is saying we want to use 64-bits to hold +@racket[0]. We have a label, which works just any other label: it +gives a symbolic name to a location in the file. + +What's happening is we are using space @emph{in the program} to hold +some data. We can reference and use that memory just like any other +memory. The space we created consists of a single word holding +@racket[0], which is found at the location named @racket['empty]. +After we set up our data section, we switch back to @racket[Text] mode +and continue with executable instructions. In this case, we use +@racket[Lea] to load the address of the @racket['empty] location, +adjusted with the @racket[type-vect] tag. In other words, we have a +vector-tagged pointer to a word of memory containing @racket[0]. In +other words, @racket[rax] contains the empty vector. + +Now you'll notice this program didn't touch the heap at all. So we +have made an empty vector without using up valuable heap space. What +remains is to adapt @racket[make-vector] so that if it's asked to +construct an empty vector it returns this canonical empty vector +rather than constructing a new one. Let's add a special case the code +above. We'll assume the canonical empty vector is statically +allocated with the label @racket['empty] somewhere (it doesn't matter +where as long as it's somewhere in the program). + +@#reader scribble/comment-reader +(ex + (eg (seq (Data) + (Label 'empty) ; canonical empty vector memory + (Dq 0) + (Text) + (Mov rax (value->bits 0)) ; length argument + (Push rax) + (Mov rax (value->bits #t)) ; init argument + ;; Start of make-vector code + (Pop r8) + (assert-natural r8) + + ; special case for length = 0 + (Cmp r8 0) + (Jne 'nonzero) + ; return canonical representation + (Lea rax (Mem 'empty type-vect)) + (Jmp 'theend) + + ;; Code for nonzero case + (Label 'nonzero) + (Mov (Mem rbx 0) r8) ; write length + (Sar r8 1) ; convert to bytes + (Mov r9 r8) ; save for heap adjustment + + ;; start initialization + (Label 'loop) + (Cmp r8 0) + (Je 'done) + (Mov (Mem rbx r8) rax) + (Sub r8 8) + (Jmp 'loop) + (Label 'done) + ;; end initialization + + (Mov rax rbx) + (Xor rax type-vect) ; create tagged pointer + (Add rbx r9) ; acct for elements and stored length + (Add rbx 8) + (Label 'theend)))) + +Here we have added a simple test at the beginning to determine if the +length argument is @racket[0]. If it is, we construct a +tagged-pointer to the canonical representation of the empty vector and +jump to the end of the instructions. If the length is non-zero, the +code jumps to the instructions we had before that does dynamic +allocation with initialization. + + +One final improvement we can make is to rework our loop knowing that +@racket[r8] will never be @racket[0] to start, shaving off a +comparison and resulting a single conditional jump at the end of the +loop: + +@#reader scribble/comment-reader +(ex + (eg (seq (Data) + (Label 'empty) ; canonical empty vector memory + (Dq 0) + (Text) + (Mov rax (value->bits 3)) ; length argument + (Push rax) + (Mov rax (value->bits #t)) ; init argument + ;; Start of make-vector code + (Pop r8) + (assert-natural r8) + + ; special case for length = 0 + (Cmp r8 0) + (Jne 'nonzero) + ; return canonical representation + (Lea rax (Mem 'empty type-vect)) + (Jmp 'theend) + + ;; Code for nonzero case + (Label 'nonzero) + (Mov (Mem rbx 0) r8) ; write length + (Sar r8 1) ; convert to bytes + (Mov r9 r8) ; save for heap adjustment + + ;; start initialization + (Label 'loop) + (Mov (Mem rbx r8) rax) + (Sub r8 8) + (Cmp r8 0) + (Jne 'loop) + ;; end initialization + + (Mov rax rbx) + (Xor rax type-vect) ; create tagged pointer + (Add rbx r9) ; acct for elements and stored length + (Add rbx 8) + (Label 'theend)))) + +And now we have essentially arrived at the code for +@racket[make-vector]. In the compiler we should generate the label +names we use, but otherwise, this is the code for +@racket[make-vector]. The canonical representation of the empty +vector can be included in the output of the top-level @racket[compile] +function. + +We are now ready to turn to strings. -One final issue for vectors is what to do about the empty vector. - -An empty vector has length zero and there are no elements contained -within it. We could reprent empty vectors the same as non-empty -vectors, although this would mean allocating a word of memory to hold -the length @racket[0] and pointing to it. This design would also have -the drawback that there could many @emph{different} empty vectors. - -Another approach is to avoid allocating memory and have a single -representation for the empty vector. One way to achieve this to -represent the address of the empty vector as the null pointer -(i.e. @racket[0]) and therefore the empty vector value is represented -by the vector type tag. Some code, such as the code to print vectors, -will need to have a special case for the empty vector to avoid a null -dereference when trying to load the length of the vector. Similarly, -there will be a special case in the implementation of -@racket[make-vector] to produce the empty vector value when given a -size of zero. This avoids allocating memory for the empty vector and -has the nice benefit that there is a unique representation of the -empty vector. @section{Representing and operating on strings} Strings will be very much like vectors---after all, they are just -another kind of array value. The key difference is that strings -are arrays not of arbitrary values, but of characters. +another kind of array value. The key difference is that strings are +arrays not of arbitrary values, but specifically of characters. +The fact that strings are @bold{homogenous arrays} of characters +will be useful in specializing the memory representation we use. + -While could use a vector to represent a string, with a unique pointer -tag, this would waste memory: every character would be allocated -64-bits of memory. Since we use unicode codepoints to represent -characters and because strings are @bold{homogenous} we need at most -21-bits to represent each character of a string. +While we could use a vector to represent a string, with a unique +pointer tag, this would waste memory: every character would be +allocated 64-bits of memory. Since we use unicode codepoints to +represent characters and because strings are @bold{homogenous} we need +at most 21-bits to represent each character of a string. There are many different representations for strings of Unicode characters, but one of the simplest is that of UTF-32. It is a @@ -281,7 +856,7 @@ memory. This would violate our assumption that the next free memory address ends in @code[#:lang "racket"]{#b000}. The solution is simple: allocate 32-bits more when the length is odd. -This sacrafices a small amount of memory in order to preserve the +This sacrifices a small amount of memory in order to preserve the invariant that allows our low-order tagging of pointers. Another complication is that we will now want to read and write @@ -290,65 +865,259 @@ units of 64-bits. We could ``fake it'' by reading and writing 64-bits at a time, carefully making sure to ignore or preserve half of the bits, however this makes the code a mess and is inefficient. -The better solution is to introduce a 32-bit register: @racket['eax]. -The @racket['eax] register is not actually a new register, but rather -is a name for the lower 32-bits of @racket['rax] (so be careful: -modifying one will change the other---they are the same register!). -Whenever @racket['eax] is used in a memory read or write, the CPU will -read or write 32-bits instead of 64. +The better solution is to use a 32-bit register: @racket[eax]. The +@racket[eax] register is not actually a new register, but rather is a +name for the lower 32-bits of @racket[rax] (so be careful: modifying +one will change the other---they are the same register!). Whenever +@racket[eax] is used in a memory read or write, the CPU will read or +write 32-bits instead of 64. So, suppose we want to create the string @racket["abc"]: @#reader scribble/comment-reader -(racketblock -(seq (Mov (Offset 'rbx 0) 3) ; write string length 3 - (Mov 'eax (char->integer #\a)) - (Mov (Offset 'rbx 8) 'eax) ; write #\a in string slot 0 - (Mov 'eax (char->integer #\b)) - (Mov (Offset 'rbx 12) 'eax) ; write #\b in string slot 1 - (Mov 'eax (char->integer #\c)) - (Mov (Offset 'rbx 16) 'eax) ; write #\c in string slot 2 - (Mov 'rax 'rbx) - (Or 'rax type-str) ; tag pointer as a string - (Add 'rbx 24)) ; advance rbx three words(!) -) +(ex + (run + (prog + (Global 'entry) + (Label 'entry) + ;; Set up heap pointer from run-time system + (Push rbx) + (Mov rbx rdi) + ;; Create string + (Mov rax (value->bits 3)) + (Mov (Mem rbx 0) rax) ; write string length 3 + (Mov eax (char->integer #\a)) + (Mov (Mem rbx 8) eax) ; write codepoint for #\a + (Mov eax (char->integer #\b)) + (Mov (Mem rbx 12) eax) ; write codepoint for #\b + (Mov eax (char->integer #\c)) + (Mov (Mem rbx 16) eax) ; write codepoint for #\c + (Mov rax rbx) + (Xor rax type-str) ; tag pointer as string + (Add rbx 24) ; acct for memory used + ;; Restore registers and return + (Pop rbx) + (Ret)))) + +At first glance, this looks remarkably similar to creating a vector, +however there are some imporant things to notice: -This looks a lot like the creation of a vector, however note that we @itemlist[ -@item{use @racket['eax] to write 32-bits of memory,} -@item{advance the offset by 4-bytes (32-bits) on each subsequent character,} -@item{write @racket[(char->integer #\a)] instead of @racket[(value->bits #\a)] into memory,} -@item{increment @racket['rbx] by 24, even though we've only written 20 bytes.} + +@item{First, this code does not use @racket[value->bits] on the +characters, but rather @racket[char->integer]. In other words, the +array here is not an array of values, but rather an array of +codepoints.} + +@item{Second, the @racket[eax] register is used to write the +codepoints to memory, which means that 32-bits (4-bytes) are written.} + +@item{Third, the offsets are growing by @racket[4] with each write, +reflecting the fact that we're only writing @racket[4] bytes per +character.} + +@item{Finally, notice that the adjustment to @racket[rbx] at the end +doesn't actually line up with how much memory was written: the code +wrote 8 bytes for the length and 3x4 bytes for the codepoints, i.e. +20 bytes, yet the heap pointer is incremented by @racket[24]. This is +to maintain our heap pointer invariant of being 8-byte aligned. It +comes at the cost of wasting 4-bytes per odd-lengthed string, which +you are seeing here.} + ] -Now let’s consider referencing elements of a string. Suppose -@racket['rax] holds a string value, we can reference an element of the -string by untagging the value and fetching from an appropriate offset. -This is just like referencing an element of a vector, except: -@itemlist[ -@item{the offset will be computed differently,} -@item{only 32-bits should be loaded from memory, and} -@item{the codepoint needs to be converted into a character.}] +Now let's set things up like we did before to be able to interactively +write examples in order to arrive at the code for +@racket[string-length], @racket[string-ref], and @racket[make-string]: -Suppose we want to fetch the 2nd element (i.e. index @racket[1]) of a -string in @racket['rax]: +@#reader scribble/comment-reader +(ex + (define (eg is) + (run + (prog + (Global 'entry) + (Extern 'raise_error) + (Label 'entry) + ;; Set up heap pointer from run-time system + (Push rbx) + (Mov rbx rdi) + ;; Create string + (Mov rax (value->bits 3)) + (Mov (Mem rbx 0) rax) ; write string length 3 + (Mov eax (char->integer #\a)) + (Mov (Mem rbx 8) eax) ; write codepoint for #\a + (Mov eax (char->integer #\b)) + (Mov (Mem rbx 12) eax) ; write codepoint for #\b + (Mov eax (char->integer #\c)) + (Mov (Mem rbx 16) eax) ; write codepoint for #\c + (Mov rax rbx) + (Xor rax type-str) ; tag pointer as string + (Add rbx 24) ; acct for memory used + + ;; include given instructions + is + + ;; Restore registers and return + (Pop rbx) + (Ret) + (Label 'err) + (Call 'raise_error))))) + + +Again, doing nothing produces the example string: + +@ex[(eg (seq))] + +The @racket[string-length] primitive is pretty-much identical to its +vector counterpart: + +@ex[(eg (seq (assert-string rax) + (Mov rax (Mem rax (- 0 type-str)))))] + + +The @racket[string-ref] primitive follows the same outline as +@racket[vector-ref], except that we compute a byte offset from the +index by shifting the to right @bold{2} places instead of 1. This is +because for an index value of @racket[_i], we want the raw bits for +@racket[(* _i 4)]. Since our encoding of integers is to multiply by +16, shifting right by 2 gives us the encoded integer times 4. We also +use @racket[eax] to load the 32-bit codepoint from memory at the end. +Finally, unlike @racket[vector-ref], once the element of the array has +been read, we have to do a little bit of work to construct the +@emph{value} that should be returned. Remember: we are storing +codepoints, not values, so @racket[string-ref] will need to construct +a character value from the codepoint, which it does by shifting and +tagging. @#reader scribble/comment-reader -(racketblock -(seq (Xor 'rax type-str) ; erase the string tag - (Mov 'eax (Offset 'rax 12)) ; load index 1 into eax - (Sal 'rax char-shift) - (Or 'rax char-type)) ; convert codepoint to character -) +(ex + (eg (seq (Push rax) + (Mov rax (value->bits 2)) ; index = 2 + ;; Start of string-ref code + (Pop r8) + (assert-natural rax) + (assert-string r8) + (Mov r9 (Mem r8 (- type-str))) + (Cmp rax r9) + (Jge 'err) + (Sar rax 2) + (Mov eax (Mem r8 rax (- 8 type-str))) + (Sal rax char-shift) + (Xor rax type-char)))) + + +Similarly, @racket[make-string] will follow a similar pattern to +@racket[make-vector]. The key differences here are: the use of +@racket[eax], shifting right by @racket[2] to compute the number of +bytes for codepoints, converting the character value in @racket[rax] +to a codepoint via a right shift, and finally an alignment to +@racket[rbx] that ensures our heap invariant is maintained: -Note the use of offset @racket[12] here: 8-bytes to skip past the -length plus 4 bytes to skip past the first character. The -@racket['eax] register is used to load 32-bits of memory, then the -value is converted to a character by shifting and tagging. +@#reader scribble/comment-reader +(ex + (eg (seq (Data) + (Label 'empty) ; canonical empty string memory + (Dq 0) + (Text) + (Mov rax (value->bits 4)) ; length argument + (Push rax) + (Mov rax (value->bits #\a)) ; init argument + ;; Start of make-string code + (Pop r8) + (assert-natural r8) + + ; special case for length = 0 + (Cmp r8 0) + (Jne 'nonzero) + ; return canonical representation + (Lea rax (Mem 'empty type-str)) + (Jmp 'theend) + + ;; Code for nonzero case + (Label 'nonzero) + + (Mov (Mem rbx 0) r8) ; write length + (Sar r8 2) ; convert to bytes + (Mov r9 r8) ; save for heap adjustment + + (Sar rax char-shift) ; convert to codepoint + + ;; start initialization + (Label 'loop) + (Mov (Mem rbx r8 4) eax) + (Sub r8 4) + (Cmp r8 0) + (Jne 'loop) + ; end initialization + + (Mov rax rbx) + (Xor rax type-str) ; create tagged pointer + (Add rbx r9) ; acct for elements and stored length + (Add rbx 8) + ;; Pad to 8-byte alignment + (Add rbx 4) + (Sar rbx 3) + (Sal rbx 3) + (Label 'theend)))) + +The alignment here is done by adding 4. If @racket[rbx] is not +currently aligned, this will bring it into alignment, making the lower +3 bits zero and then the shift right and left have no impact. On the +other hand, if @racket[rbx] is already aligned, the @racket[Add] simply sets +the third bit, which is then stripped off by the shifts. + + +The handling of the empty string is just like that of the empty +vector. In fact, if you look at the data section, it's +@emph{identical} to the data section for the empty vector. This +suggests that both the empty string and the empty vector can share the +same static memory for their canonical representation. This doesn't +mean the empty string and the empty vector are the same value, they +are not. But they @emph{point} to the same memory. One points with +string tag and the other points with a vector tag! + +@section{Compiling string literals} + +The final issue to address is how to compile string literals such as +@racket["fred"]. For this we can turn to the mechanism of static +memory that we saw earlier with the empty vector and empty string. +Since a string literal is totally determined at compile-time, we can +avoid allocation in the heap and instead allocate in the program +itself. For example, it's easy to construct @racket["abc"] using +static memory: -Just as we did with vectors, we want the compiler to emit code that -checks indices are inbound for the string. +@ex[ +(eg (seq (Data) + (Label 'abc) + (Dq (value->bits 3)) + (Dd (char->integer #\a)) + (Dd (char->integer #\b)) + (Dd (char->integer #\c)) + (Text) + (Lea rax (Mem 'abc type-str))))] + +Notice that this uses the @racket[Dd] psuedo-instruction for +specifying 32-bit quantities in the data section. This lays out a +sized array of codepoints in the data section and the @racket[Lea] +instruction constructs a string-tagged pointer to it. + +Using this idea we can formulate a compiler for string literals: +@ex[ +(define (compile-string s) + (let ((l (gensym 'string)) + (n (string-length s))) + (match s + ["" (seq (Lea rax (Mem 'empty type-str)))] + [_ + (seq (Data) + (Label l) + (Dq (value->bits n)) + (compile-string-chars (string->list s)) + (if (odd? n) (Dd 0) (seq)) + (Text) + (Lea rax (Mem l type-str)))]))) +] @section{A Compiler for @this-lang} diff --git a/www/notes/hustle.scrbl b/www/notes/hustle.scrbl index 8fa4100d..c8841a12 100644 --- a/www/notes/hustle.scrbl +++ b/www/notes/hustle.scrbl @@ -4,20 +4,16 @@ @(require redex/pict racket/runtime-path scribble/examples - (except-in hustle/semantics ext lookup) - (prefix-in sem: (only-in hustle/semantics ext lookup)) "../fancyverb.rkt" "utils.rkt" "ev.rkt" - "../utils.rkt") + "../utils.rkt" + "diagrams.rkt" + hustle/types) @(define codeblock-include (make-codeblock-include #'h)) -@(ev '(require rackunit a86)) -@(ev `(current-directory ,(path->string (build-path langs "hustle")))) -@(void (ev '(with-output-to-string (thunk (system "make runtime.o"))))) -@(for-each (λ (f) (ev `(require (file ,f)))) - '("main.rkt" "heap.rkt" "unload.rkt" "interp-prims-heap.rkt")) +@(ev '(require rackunit a86 hustle hustle/unload hustle/heap hustle/interp-prims-heap hustle/compile-ops)) @(define this-lang "Hustle") @(define prefix (string-append this-lang "-")) @@ -86,20 +82,6 @@ These features will operate like their Racket counterparts: OCaml's @tt{ref} type, but we will examine this aspect later. For now, we treat boxes as immutable data structures.} -We will also add support for writing pair and box @emph{literals} -using the same @racket[quote] notation that Racket uses. - -These features will operate like their Racket counterparts: -@ex[ -(unbox '#&7) -(car '(3 . 4)) -(cdr '(3 . 4)) -(box? '#&7) -(cons? '(3 . 4)) -(box? '(3 . 4)) -(cons? '#&7) -] - @section[#:tag-prefix prefix]{Empty lists can be all and end all} While we've introduced pairs, you may wonder what about @emph{lists}? @@ -120,146 +102,79 @@ We use the following AST data type for @|this-lang|: @filebox-include-fake[codeblock "hustle/ast.rkt"]{ #lang racket ;; type Expr = ... | (Lit Datum) -;; type Datum = ... | (cons Datum Datum) | (box Datum) | '() +;; type Datum = ... | '() ;; type Op1 = ... | 'box | 'car | 'cdr | 'unbox | 'box? | 'cons? ;; type Op2 = ... | 'cons } @section[#:tag-prefix prefix]{Parsing} -Mostly the parser updates for @|this-lang| are uninteresting. The -only slight twist is the addition of compound literal datums. - -It's worth observing a few things about how @racket[quote] works in -Racket. First, some datums are @emph{self-quoting}, i.e. we can -write them with or without quoting and they mean the same thing: -@ex[ -5 -'5] - -All of the datums consider prior to @|this-lang| have been self-quoting: -booleans, integers, and characters. - -Of the new datums, boxes are self-quoting, but pairs and the empty -list are not. -@ex[ -#&7 -'#&7 -(eval:error ()) -'() -(eval:error (1 . 2)) -'(1 . 2)] - -The reason for this is that unquoted list datums would be confused -with expression forms without the @racket[quote], so its required, -however for the other datums, there's no possible confusion and the -@racket[quote] is inferred. Note also that once inside a self-quoting -datum, it's unambiguous that we're talking about literal data and not -expressions that need to be evaluated, so you can have empty lists and -pairs: -@ex[ -#&() -#&(1 . 2)] - -This gives rise to two notions of datums that our parser uses, -with (mutually defined) predicates for each: - -@filebox-include-fake[codeblock "hustle/parse.rkt"]{ -;; Any -> Boolean -(define (self-quoting-datum? x) - (or (exact-integer? x) - (boolean? x) - (char? x) - (and (box? x) (datum? (unbox x))))) - -;; Any -> Boolean -(define (datum? x) - (or (self-quoting-datum? x) - (empty? x) - (and (cons? x) (datum? (car x)) (datum? (cdr x))))) -} - -Now when the parser encounters something that is a self-quoting datum, -it can parse it as a @racket[Lit]. But for datums that are quoted, it -will need to recognize the @racket[quote] form, so anything that has -the s-expression shape @racket[(quote d)] will also get parsed as a -@racket[Lit]. +Mostly the parser updates for @|this-lang| are uninteresting. We've +added some new unary and binary primitive names that the parser now +recognizes for things like @racket[cons], @racket[car], +@racket[cons?], etc., however, one wrinkle is that we now have a very +limited form of @racket[quote], so it's worth discussing what this +means for the concrete syntax of our language. + +@subsection{Quote and the notion of self-quoting datums} + +In Racket, some datums are @emph{self-quoting}, which means they don't +need to be quoted. For example, @racket[5] is a self-quoting datum. +You actually can quote @racket[5]: @racket['5]. This means exactly +the same thing as @racket[5]. These are two different concrete +syntaxes for exactly the same thing: the integer literal @racket[5]. + +The whole reason for @racket[quote] is that it is used to indicate +when something is a datum when it would otherwise be interpreted as an +expression. This becomes relevant we start having datums with +parentheses in them because for example the expression @racket[(add1 +5)] and the datum @racket['(add1 5)] mean very different things, +whereas @racket[5] and @racket['5], do not. So lists (and pairs) are +@emph{not} self-quoting: we are required to use @racket[quote] when we +want to write such datums: @racket[(code:quote (add1 5))], which can +also be written in shorthand form as @racket['(add1 5)]. + +Up until this point, all of our datums have been self-quoting and we +have thus just left @racket[quote] out of the concrete syntax. Now we +have @racket[quote] for the empty list. + +@subsection{Parsing quoted datums and self-quoting datums} + +In our parser, we introduce a predicate for identifying self-quoting +datums (@racket[self-quoting-datum?]), which includes integers, +boolean, and characters; and another for identifying the larger class +of datums (@racket[datum?]), which includes all the self-quoting +datums, plus the empty list @racket['()]. Notice in the parser that +self-quoting datums are expressions, but quoted datums must occur +withing a @racket[(code:quote ...)] form. -Things can get a little confusing here so let's look at some examples: @ex[ (parse 5) -(parse '5) +(code:comment "This is quoting at the Racket-level and the parser sees 5") +(parse '5) +(code:comment "This is quoting at the Hustle-level and the parser see '(quote 5)") +(code:comment "which it parses at (Lit 5)") +(parse ''5) +(code:comment "This is quoting at the Racket-level and the parser see '(),") +(code:comment "but () is not valid expression syntax, hence a parse error") +(eval:error (parse '())) +(code:comment "This is quoting at the Hustle-level and the parser sees '(quote ()),") +(code:comment "which is parsed as the empty list.") +(parse ''()) ] -Here, both examples are really the same. When we write @racket['5], -that @racket[read]s it as @racket[5], so this is really the same -example and corresponds to an input program that just contains the -number @racket[5] and we are calling @racket[parse] with an argument -of @racket[5]. - -If the input program contained a quoted @racket[5], then it would be -@racket['5], which we would represent as an s-expression as -@racket[''5]. Note that this reads as @racket['(quote 5)], i.e. a -two-element list with the symbol @racket['quote] as the first element -and the number @racket[5] as the second. So when writing examples -where the input program itself uses @racket[quote] we will see this -kind of double quotation, and we are calling @racket[parse] with -a two-element list as the argument: - -@margin-note{FIXME: langs needs to be update to parse this correctly.} +It's worth noting that while we have added pairs and boxes to our +language, we have not added @emph{literal notation} for these things. +(We will eventually.) So things like @racket['(1 . 2)] are not valid +syntax in Hustle: @ex[ -(eval:error (parse ''5))] - -This is saying that the input program was @racket['5]. Notice that it -gets parsed the same as @racket[5] by our parser. - -If we were to parse the empty list, this should be considered a parse -error because it's like writing @racket[()] in Racket; it's not a valid -expression form: - -@ex[ -(eval:error (parse '()))] - -However, if the empty list is quoted, i.e. @racket[''()], then we are -talking about the expression @racket['()], so this gets parsed as -@racket[(Lit '())]: - -@ex[ -(parse ''())] - -It works similarly for pairs: - -@margin-note{FIXME: langs needs to be update to parse second example correctly.} +(eval:error (parse ''(1 . 2))) +] -@ex[ -(eval:error (parse '(1 . 2))) -(eval:error (parse ''(1 . 2)))] - -While these examples can be a bit confusing at first, implementing -this behavior is pretty simple. If the input is a -@racket[self-quoting-datum?], then we parse it as a @racket[Lit] -containing that datum. If the the input is a two-element list of the -form @racket[(list 'quote _d)] and @racket[_d] is a @racket[datum?], -the we parse it as a @racket[Lit] containing @racket[_d]. - -Note that @emph{if} the examples are confusing, the parser actually -explains what's going on in Racket. Somewhere down in the code that -implements @racket[read] is something equivalent to what we've done -here in @racket[parse] for handling self-quoting and explicitly quoted -datums. Also note that after the parsing phase, self-quoting and -quoted datums are unified as @racket[Lit]s and we no longer need to be -concerned with any distinctions that existed in the concrete syntax. - -The only other changes to the parser are that we've added some new -unary and binary primitive names that the parser now recognizes for -things like @racket[cons], @racket[car], @racket[cons?], etc. @codeblock-include["hustle/parse.rkt"] - - - @section[#:tag-prefix prefix]{Meaning of @this-lang programs, implicitly} To extend our interpreter, we can follow the same pattern we've been @@ -280,16 +195,12 @@ Using their Racket counterparts of course! @codeblock-include["hustle/interp-prim.rkt"] -@margin-note{FIXME} - We can try it out: @ex[ (interp (parse '(cons 1 2))) (interp (parse '(car (cons 1 2)))) (interp (parse '(cdr (cons 1 2)))) -(eval:error (interp (parse '(car '(1 . 2))))) -(eval:error (interp (parse '(cdr '(1 . 2))))) (interp (parse '(let ((x (cons 1 2))) (+ (car x) (cdr x))))) ] @@ -527,143 +438,411 @@ the final answer from the result: @section[#:tag-prefix prefix]{Representing @this-lang values} -The first thing do is make another distinction in the kind of values -in our language. Up until now, each value could be represented in a -register. We now call such values @bold{immediate} values. +Since we have grown the set of values in our langauge, we have to +address the issue of how this set of values can be represented at +run-time. The new values we've added are: the empty lists, pairs, and +boxes. Of these, the empty list is straightfoward: it can be +represented like any of our other enumerated values: pick an unused +bit pattern and designate it as representing the empty list. Boxes +and pairs will require some new mechanisms. + +@subsection{The need for memory} + +There's an obvious conundrum encountered as soon as you start thinking +about representing Hustle values. Remember that values have to fit in +a register, i.e. we have at most 64 bits to represent all of our +values. We've gotten by so far by using some small number of bits to +encode the type of the value and the remaining bits to represent the +value itself. We now have new kinds of values: pairs and boxes. +These are distinct from the existing types of values, so we will need +to devote some bits to indicating these new types. But what about the +value part? Well a pair contains two values, e.g. @racket[(cons 1 2)] +contains both the value @racket[1] and the value @racket[2]. How can +we possibly fit both, which each may take up 64 bits, into a single +64-bit register? Even if we were to chop integers down to say 30 bits +so that two could fit in a word with type tag bits left over, that +only works for @emph{pairs of integers}, and even then it only works +for small integers. You might also be tempted to use more registers +to represent values. Perhaps we use two registers to store values. +One is unused except in the case of pairs, then each register holds +the elements of the pair. + +But the true power of these new kinds of values is the ability to +construct @bold{arbitrarily large collections of data}. That power +comes from being able to construct a pair of @emph{any} two values, +@emph{including other pairs}. So while we might be able to find ways +of encoding small collections, we have to face the fact that we might +construct large collections and that no strategy that depends upon a +fixed number of values will suffice. + + +We need memory. When creating a pair, just like in our explicit +heap-based interpreter, we need to allocate memory and think of the +pair value as having a @emph{pointer} to that memory. In this way we +can construct arbitrarily large collections of values, bound only by +the available memory on our system. + + +@subsection{Tagged pointer values} + +That idea that we can use pointers to memory to represent datatypes +like boxes and pairs seems simple enough, but we also still have to +deal with the other aspects of our value encoding. Namely, we need to +be able to distinguish all of the disjoint datatypes in our language. +A pointer is seemingly just an arbitrary 64-bit integer. How can we +tell a pointer apart from other bit patterns that represent integers, +booleans, characters, etc.? -We introduce a new category of values which are @bold{pointer} values. -We will (for now) have two types of pointer values: boxes and pairs. +A memory location is represented (of course, it's all we have!) as a +number. The number refers to some address in memory. On an x86 +machine, memory is @bold{byte-addressable}, which means each address +refers to a 1-byte (8-bit) segment of memory. If you have an address +and you add 1 to it, you are refering to memory starting 8-bits from the +original address. + +It's tempting to follow the approach we've already used: shift and +tag. In other words, take a pointer, shift it to the left some number +of bits and tag the lower bits with a unique pattern to indicate the +type as being either a pair or a box. + +This worked for things like booleans, characters, eof, void, +etc. because we chould shift to the left without losing any +information. In the case of integers, we @emph{did} lose some +information: we cut down the range of integer values that are +representable in our language. But the integers that were left still +made sense. + +Unfortunately pointers don't work that way. If we shift a pointer and +bits fall off, we're no longer pointing at the same memory location. +So what are we to do? + +There are many options, but we adopt a simple approach. It starts +from the observation that we will always allocate memory in multiples +of 8 bytes. So if our memory starts out aligned to 8 bytes, then all +of the addresses we will reprent will also be aligned to 8 bytes. +That means addresses will always end in @binary[#b000 3]. We can +therefore use these bits to store information @emph{without losing any +information about the address itself}! + +The first thing to do is make another distinction in the kind of +values in our language. Up until now, each value could be represented +in a register alone. We now call such values @bold{immediate} values. + +We introduce a new category of values which are @bold{tagged pointer} +values. Tagged pointers also fit into registers, but they refer to memory +so they cannot be understood by the contents of the register alone; we have to +take the memory into consideration too. + +We will (for now) have two types of tagged pointer values: boxes and +pairs. So we now have a kind of hierarchy of values: -@verbatim{ -- values - + pointers (non-zero in last 3 bits) - * boxes - * pairs - + immediates (zero in last three bits) - * integers - * characters - * booleans - * ... -} +@itemlist[ +@item{Values + @itemlist[@item{Tagged pointers (non-zero in last three bits) + @itemlist[@item{Boxes} @item{Pairs}]}] + @itemlist[@item{Immediates (zero in last three bits) + @itemlist[@item{Integers} @item{Characters} @item{Booleans} @item{...}]}]}] -We will represent this hierarchy by shifting all the immediates over 3 -bits and using the lower 3 bits to tag things as either being -immediate (tagged @code[#:lang "racket"]{#b000}) or a box or pair. -To recover an immediate value, we just shift back to the right 3 bits. +We will represent this hierarchy by shifting all the immediates over +@number->string[imm-shift] bits and using the lower +@number->string[imm-shift] bits to tag things as either being +immediate (tagged @binary[0 imm-shift]) or a box or pair. To recover +an immediate value, we just shift back to the right +@number->string[imm-shift] bits. -The pointer types will be tagged in the lowest three bits. A box -value is tagged @code[#:lang "racket"]{#b001} and a pair is tagged -@code[#:lang "racket"]{#b010}. The remaining 61 bits will hold a -pointer, i.e. an integer denoting an address in memory. +So for example: + +@(define (val-eg v) + @item{the value @racket[#,v] is represented by the bits @binary[(value->bits v) imm-shift], aka @racket[#,(value->bits v)].}) + +@itemlist[ + @val-eg[#t] + @val-eg[#f] + @val-eg[0] + @val-eg[1] + @val-eg[5] + @val-eg[#\a] + @val-eg[#\b] +] + +The pointer types will be tagged in the lowest +@number->string[imm-shift] bits. A box value is tagged +@binary[type-box imm-shift] and a pair is tagged @binary[type-cons +imm-shift]. The remaining @number->string[(- 64 imm-shift)] bits will +hold a pointer, i.e. an integer denoting an address in memory. To +obtain the address, no shifting is done; instead we simply zero-out +the tag. The idea is that the values contained within a box or pair will be -located in memory at this address. If the pointer is a box pointer, -reading 64 bits from that location in memory will produce the boxed -value. If the pointer is a pair pointer, reading the first 64 bits -from that location in memory will produce one of the value in the pair -and reading the next 64 bits will produce the other. In other words, -constructors allocate and initialize memory. Projections dereference -memory. - -The representation of pointers will follow a slightly different scheme -than that used for immediates. Let's first talk a bit about memory -and addresses. +located in memory at this address. If the tagged pointer is a box +pointer, reading 64 bits from that location in memory will produce the +boxed value. If the pointer is a pair pointer, reading the first 64 +bits from that location in memory will produce one of the value in the +pair and reading the next 64 bits will produce the other. In other +words, constructors allocate and initialize memory. Projections +dereference memory. + +It's more difficult to construct examples of tagged pointer values +because a value's representation now depends on what's in memory. +Moveover, a value's representation is no longer unique. We can no +longer say things like ``the value @racket['(1 . 2)] is represented by +the bits...'' because there are many possible bits that could +represent this value. + +We can however say that if a memory address holds two consecutive +values: 1 and 2, then a value @racket['(1 . 2)] may be represented by +the bits you get when you tag that pointer as a pair. Let's use this +idea to write some representation examples. These are all stated +hypothetically based on what has to be in memory: -A memory location is represented (of course, it's all we have!) as a -number. The number refers to some address in memory. On an x86 -machine, memory is @bold{byte-addressable}, which means each address -refers to a 1-byte (8-bit) segment of memory. If you have an address -and you add 1 to it, you are refering to memory starting 8-bits from the -original address. +@itemlist[ +@item{if address 0 holds the value @racket[#t] and address 8 holds the value @racket[#f], + then the value @racket['(#t . #f)] may be represented by the bits @binary[type-cons], + aka @racket[#,type-cons].}] -We will make a simplifying assumption and always store things in -memory in multiples of 64-bit chunks. So to go from one memory -address to the next @bold{word} of memory, we need to add 8 (1-byte -times 8 = 64 bits) to the address. +This a perfectly valid, if somewhat unrealistic example. Your +operating system is likely not going to let you use address 0 (or 8 +for that matter). That's OK. We don't really care what the actual +address is @emph{so long as it's always divisible by 8}. That's the +only thing our encoding scheme depends upon. -What is 8 in binary? @code[#:lang "racket"]{#b1000} +Let's try another example: -What's nice about this is that if we start from a memory location that -is ``word-aligned,'' i.e. it ends in @code[#:lang "racket"]{#b000}, -then every 64-bit index also ends in @code[#:lang "racket"]{#b000}. +@itemlist[ +@item{if address 98760 holds the value @racket[#t] and address 98768 holds the + value @racket[#f], then the value @racket['(#t . #f)] may be represented by the + bits @binary[(+ 98760 type-cons)], aka @racket[#,(+ 98760 type-cons)].}] -What this means is that @emph{every} address we'd like to represent -has @code[#:lang "racket"]{#b000} in its least signficant bits. We -can therefore freely uses these three bits to tag the type of the -pointer @emph{without needing to shift the address around}. If we -have a box pointer, we can simply zero out the box type tag to obtain -the address of the boxes content. Likewise with pairs. +A couple things to notice in this example: (1) the address is +divisible by 8, (2) the representation of the value @racket['(#t +. #f)] @emph{is not}. That's because we tacked on the cons type tag +in unused bits of the pointer. We haven't lost any information +though: to recover the pointer, simply erase the tag (either by +or-ing, xor-ing, or subtracting the tag from the tagged pointer; they +are all equivalent if the address is divisible by 8 and the tag is +less than 8, which it is). +So in general, we have: + +@itemlist[ + +@item{if address @racket[_a] is divisible by 8 and holds the value + @racket[_v₁] and address @racket[_a] + 8 holds the value @racket[_v₂], + then the value @racket[(cons _v₁ _v₂)] may be represented by the bits + @racket[(bitwise-xor _a #,(binary type-cons))].} + +@item{if address @racket[_a] is divisible by 8 and holds the value + @racket[_v], + then the value @racket[(box _v)] may be represented by the bits + @racket[(bitwise-xor _a #,(binary type-box))].} + +] -We use a register, @racket['rbx], to hold the address of the next free -memory location in memory. To allocate memory, we simply increment -the content of @racket['rbx] by a multiple of 8. To initialize the -memory, we just write into the memory at that location. To construct a -pair or box value, we just tag the unused bits of the address. +@(define ra (+ (random 0 10000) 123456)) + +We can also turn things around: + +@itemlist[ @item{if bits @binary[(+ ra type-cons)] (aka @racket[#,(+ +ra type-cons)]) represents a value, then at address @binary[ra] (aka +@racket[#,ra]) there is some value @racket[_v₁] and at @binary[(+ ra +8)] (aka @racket[#,(+ ra 8)]) there is some value @racket[_v₂].}] + +We know this because the bits end in the pair type tag, thus the value +represented is a pair and it must be the case that there are two +values at the address encoded in the bits. + +In general: + +@itemlist[ @item{if bits @racket[_b] represents a value and @racket[(= +(bitwise-and _b #,(binary ptr-mask)) #,(binary type-cons))], then at +address @racket[(bitwise-xor _b #,(binary type-cons))] there is some +value @racket[_v₁] and at address @racket[(+ (bitwise-xor _b #,(binary +type-cons)) 8)] there is some value @racket[_v₂].} +@item{if bits @racket[_b] represents a value and @racket[(= +(bitwise-and _b #,(binary ptr-mask)) #,(binary type-box))], then at +address @racket[(bitwise-xor _b #,(binary type-box))] there is some +value @racket[_v].}] + + +OK, we now have a good model of how these new kinds of values +can be @emph{represented}, but how can we actually construct +and manipulate them? + + +@subsection{A source of free memory} + +We've established how we can use memory to represent boxes and +pairs, but it remains to be seen where this memory comes from and how +we can use it to construct these kinds of values. + +So far, our only ability to allocate memory has come from using the +stack. When we push variable bindings or the results of intermediate +computations on the stack, we ``allocate'' memory by using more of the +stack space. When we pop these values off, we ``deallocate'' that +memory by making it available to be overwritten by future stack +pushes. + +Since it seems to be the only game in town, it's obviously tempting to +use the stack to allocate pairs and boxes. But here's the rub: +variable bindings and intermediate results follow a straightforward +stack discipline that we can read off from the text of a program. For +example, in @racket[(let ((_x _e₁)) _e₂)], we know that we can push +@racket[_e₁]'s value on the stack before executing the code for +@racket[_e₂] and then pop it off at the end of the instructions for +the whole @racket[let]. Likewise in @racket[(+ _e₁ _e₂)], we can push +@racket[_e₁]'s value on the stack while computing @racket[_e₂] and +then pop it off to do the addition. But with @racket[(cons _e₁ _e₂)], +if pushed the values of @racket[_e₁] and @racket[_e₂] on the stack and +then made a tagged pointer to that value, @emph{when} would we pop it +off? Definitely not at the end of the @racket[(cons _e₁ _e₂)] +expression because after all we need to be able to access the parts of +the pair after making it; deallocating then would construct and +immediate destroy the pair. On the other hand, @emph{not} popping at +the end of the expression destroys one of our compiler invariants +which is that, by the time we get to the end of the instructions for +the compiled code of an expression, we have restored the stack to +whatever state it was in at the start. If we destroy that, how will +variables and binary operations work? Notice the shape of the stack +could no longer be read off from the text of a program. Consider: + +@racketblock[ +(let ((x (if (zero? (read-byte)) (cons 1 2) #f))) + x) +] +Where is @racket[x]'s value on the stack? If we allocate pairs on the +stack, there may or may not be a pair sitting before it @emph{and +there's no way to know at compile-time}. Good luck compiling that +variable occurrence! + +So... the stack is out. Mostly this is because the lifetime of +pointer values is not lexical: it's not a property of the text of a +program, but rather its execution. So we will need another source +of free memory; memory that can outlive elements on the stack. +We'll call this memory the @bold{heap}. + +For this, we will turn to our run-time system. Before it calls the +compiled code of a program, we will have it allocate a chunk of memory +and pass it as an argument to the compiled code. The compiled code +will then install that pointer into a designated register, much like +how @racket[rsp] is designated to hold the stack pointer. So instead +of doing this in the @tt{main} entry point of the run-time: + +@verbatim|{ + val_t result; + result = entry(); + print_result(result); +}| + +We'll update it to: + +@verbatim|{ + heap = malloc(8 * heap_size); // allocate heap + val_t result; + result = entry(heap); // pass in pointer to heap + print_result(result); +}| + +Here we are allocating some number of words of memory (how many words +is given by the constant @tt{heap_size}), via @tt{malloc} and then +calling the compiled code with an argument which is the pointer to +this freshly allocated memory. The compiled code can hold on to a +pointer to this memory and write into it in to allocate new pairs and +boxes. + +We will designate the @racket[rbx] register as the heap pointer +register. This is an arbitrary choice, other than the fact that we +selected a callee-saved (aka non-volatile) register. This is useful +for us because we need the heap pointer to be preserved across calls +into the run-time system. Had we designated a caller-save (volatile) +register, we'd need to save and restore it ourselves before and after +@emph{every} call. Choosing a non-volatile register does mean we have +to save and restore the @emph{caller's} @racket[rbx], but we do this +just once at the beginning and end of the program. + +So our top-level compiler looks like this: -The generated code will have to coordinate with the run-time system to -initialize @racket['rbx] appropriately, which we discuss in -@secref["hustle-run-time"]. +@#reader scribble/comment-reader +(racketblock +;; ClosedExpr -> Asm +(define (compile e) + (prog (Global 'entry) + (Label 'entry) + ... + (Push rbx) ; save the caller's register + (Mov rbx rdi) ; install heap pointer + (compile-e e '()) ; run! + (Pop rbx) ; restore caller's register + ... + (Ret)))) + + +Now @racket[compile-e] can produce code that uses the heap pointer in +register @racket[rbx]. If we want to use some of this memory we can +write into it, adjust @racket[rbx] by the amount we just used, and +then produce tagged pointers to the location we wrote to. So for example the following creates a box containing the value 7: @#reader scribble/comment-reader (racketblock -(seq (Mov 'rax (value->bits 7)) - (Mov (Offset 'rbx 0) 'rax) ; write '7' into address held by rbx - (Mov 'rax 'rbx) ; copy pointer into return register - (Or 'rax type-box) ; tag pointer as a box - (Add 'rbx 8)) ; advance rbx one word +(seq (Mov rax (value->bits 7)) + (Mov (Mem rbx 0) rax) ; write '7' into address held by rbx + (Mov rax rbx) ; copy pointer into return register + (Xor rax type-box) ; tag pointer as a box + (Add rbx 8)) ; advance rbx one word ) -If @racket['rax] holds a box value, we can ``unbox'' it by erasing the + + + +If @racket[rax] holds a box value, we can ``unbox'' it by erasing the box tag, leaving just the address of the box contents, then dereferencing the memory: @#reader scribble/comment-reader (racketblock -(seq (Xor 'rax type-box) ; erase the box tag - (Mov 'rax (Offset 'rax 0))) ; load memory into rax +(seq (Xor rax type-box) ; erase the box tag + (Mov rax (Mem rax 0))) ; load memory into rax +) + +As a slight optimization, instead of doing a run-time tag erasure, we +can simply adjust the offset by the tag quantity so that reading the +contents of a box (or any other pointer value) is a single +instruction: + +@#reader scribble/comment-reader +(racketblock +(seq (Mov rax (Mem rax (- type-box)))) ; load memory into rax ) Pairs are similar, only they are represented as tagged pointers to two -words of memory. Suppose we want to make @racket[(cons 3 4)]: +words of memory. Suppose we want to make @racket[(cons #t #f)]: @#reader scribble/comment-reader (racketblock -(seq (Mov 'rax (value->bits 4)) - (Mov (Offset 'rbx 0) 'rax) ; write '4' into address held by rbx - (Mov 'rax (value->bits 3)) - (Mov (Offset 'rbx 8) 'rax) ; write '3' into word after address held by rbx - (Mov 'rax rbx) ; copy pointer into return register - (Or 'rax type-cons) ; tag pointer as a pair - (Add 'rbx 16)) ; advance rbx 2 words +(seq (Mov rax (value->bits #t)) + (Mov (Mem rbx 0) rax) ; write '#t' into address held by rbx + (Mov rax (value->bits #f)) + (Mov (Mem rbx 8) rax) ; write '#f' into word after address held by rbx + (Mov rax rbx) ; copy pointer into return register + (Xor rax type-cons) ; tag pointer as a pair + (Add rbx 16)) ; advance rbx 2 words ) This code writes two words of memory and leaves a tagged pointer in -@racket['rax]. It's worth noting that we chose to write the -@racket[cdr] of the pair into the @emph{first} word of memory and the -@racket[car] into the @emph{second}. This may seem like a strange -choice, but how we lay out the memory is in some sense an arbitrary -choice, so long as all our pair operations respect this layout. We -could have just as easily done the @racket[car] first and @racket[cdr] -second. The reason for laying out pairs as we did will make things -slightly more convenient when implementing the @racket[cons] primitive -as we'll see later. - +@racket[rax]. -If @racket['rax] holds a pair value, we can project out the elements -by erasing the pair tag, leaving just the address of the pair contents, -then dereferencing either the first or second word of memory: +If @racket[rax] holds a pair value, we can project out the elements by +erasing the pair tag (or adjusting our offset appropriately) and +dereferencing either the first or second word of memory: @#reader scribble/comment-reader (racketblock -(seq (Xor 'rax type-cons) ; erase the pair tag - (Mov 'rax (Offset 'rax 8)) ; load car into rax - (Mov 'rax (Offset 'rax 0))) ; or... load cdr into rax +(seq (Mov rax (Mem rax (- 0 type-cons))) ; load car into rax + (Mov rax (Mem rax (- 8 type-cons)))) ; or... load cdr into rax ) From here, writing the compiler for @racket[box], @racket[unbox], @@ -671,131 +850,551 @@ From here, writing the compiler for @racket[box], @racket[unbox], putting together pieces we've already seen such as evaluating multiple subexpressions and type tag checking before doing projections. +@subsection{Making examples} + +It's more challenging to use @racket[asm-interp] to actually execute +examples since we need some coordination between the run-time system +and @racket[asm-interp] in order to allocate the heap, but for the +moment, let's see how we can effectively work around this coordination +to make examples that actually run without using the run-time system. + + +An alternative to asking the run-time system to allocate memory (which +in turn asks our operating system to allocate memory), we can instead +bake some memory into the text of our assembly program itself. This +ends up as space @emph{in the object file} of our compiled and +assembled program that also holds the instructions for our code. To +do this we can create a @bold{data section} in our code. Up until now +our assembly programs have lived in the @bold{text section}, which is +the part that holds instructions to be executed. In contrast, the +data section just holds data, not instructions. When the operating +system runs our program, it loads the object file into memory, so +anything we put into the data section (as well as all of our +instructions) are in memory and we can use this memory. Fundamentally +it's no different from the memory allocated by @tt{malloc} in our +run-time system. The key difference is that this space comes from the +object file and therefore has to be determine at compile-time rather +than at run-time using @tt{malloc}. Hence it is referred to as +@bold{static memory}. + +When constructing an assembly program, we can switch the data section +by using the @racket[(Data)] psuedo-instruction. What this means is +that the instructions that follow should be assembled into the data +part of the file and not the text (code) part. To switch back to the +text section, use the @racket[(Text)] directive. Within the data +section we can use the @racket[Dq] ``instruction'' to designate one +(64-bit) word of static memory. It's not actually an instruction +(hence the scare-quotes) because it doesn't execute; instead it says +put these bits at this spot in the program. So this sequence: + +@racketblock[ +(seq (Data) + (Dq 1) + (Dq 2) + (Dq 3))] + +is saying that in the data section there should be three words of +memory containing the bits @racket[1], @racket[2], and @racket[3], +respectively. + +If we want to get a pointer to this memory, we need to name the +location with a label and then use the @racket[Lea] instruction to +load it's address into a register at run-time: + + +@racketblock[ +(seq (Data) + (Label 'd) + (Dq 1) + (Dq 2) + (Dq 3) + (Text) + (Lea rax 'd))] + +Let's try it out: + +@ex[ +(asm-interp + (prog (Global 'entry) + (Label 'entry) + (Data) + (Label 'd) + (Dq 1) + (Dq 2) + (Dq 3) + (Text) + (Lea rax 'd) + (Ret)))] + +Now, what we get back is the @emph{address} of that memory. It's some +arbitrary number (but notice what it is divisible by!). + +If we'd like we can dereference that memory to fetch the contents: + +@ex[ +(asm-interp + (prog (Global 'entry) + (Label 'entry) + (Data) + (Label 'd) + (Dq 1) + (Dq 2) + (Dq 3) + (Text) + (Lea rax 'd) + (Mov rax (Mem rax 0)) + (Ret)))] + + +@;{ + +If we changed the offset to @racket[8], we'd get @racket[2]; +@racket[16] would get @racket[3]. We have essentially created a +little static array. We can also write into that array: + +@ex[ +(asm-interp + (prog (Global 'entry) + (Label 'entry) + (Data) + (Label 'd) + (Dq 1) + (Dq 2) + (Dq 3) + (Text) + (Lea rax 'd) + (Mov r8 100) + (Mov (Mem rax 0) r8) + (Mov rax (Mem rax 0)) + (Ret)))] + +So we can use this as a basis for making little executable examples to +run our compiler. The idea is we can @emph{statically} allocate heap +space and then use that to execute code. + +Here's a little helper function for (statically) allocating a given +number of words and loading a pointer to it into a given register: + +@#reader scribble/comment-reader +(ex +;; Statically allocate i words of memory and +;; set register r to its address +;; Reg Nat -> Asm +(define (alloc r i) + (let ((l (gensym 'data))) + (seq (Data) + (Label l) + (make-list i (Dq 0)) + (Text) + (Lea r (Mem l))))) +) + +} + +To do that, let's just call @tt{malloc} ourselves! + +@ex[ +(asm-interp + (prog (Global 'entry) + (Extern 'malloc) + (Label 'entry) + (Sub rsp 8) + (Mov rdi (* 10 8)) + (Call 'malloc) + (Add rsp 8) + (Ret)))] + +This sets up a call to @tt{malloc(8*10)}, which allocates 10 words and +returns the pointer in @racket[rax]. + +OK, let's make a pair: + +@#reader scribble/comment-reader +(ex +(eval:alts + (asm-interp + (prog (Global 'entry) + (Extern 'malloc) + (Label 'entry) + (Push rbx) + (Mov rdi (* 10 8)) + (Call 'malloc) + (Mov rbx rax) + (Mov rax (value->bits #t)) + (Mov (Mem rbx 0) rax) ; write #t + (Mov rax (value->bits #f)) + (Mov (Mem rbx 8) rax) ; write #f + (Mov rax rbx) ; copy pointer + (Xor rax type-cons) ; tag as pair + (Add rbx 16) ; account for memory used + (Pop rbx) + (Ret))) + (begin + (define this + (asm-interp + (prog (Global 'entry) + (Extern 'malloc) + (Label 'entry) + (Push rbx) + (Mov rdi (* 10 8)) + (Call 'malloc) + (Mov rbx rax) + (Mov rax (value->bits #t)) + (Mov (Mem rbx 0) rax) ; write #t + (Mov rax (value->bits #f)) + (Mov (Mem rbx 8) rax) ; write #f + (Mov rax rbx) ; copy pointer + (Xor rax type-cons) ; tag as pair + (Add rbx 16) ; account for memory used + (Pop rbx) + (Ret)))) + this))) + + +This @emph{should} create a pair that is represented in memory like +this: + +@make-heap-diagram['((cons 0) #t #f)] + +What we get is @racket[#,(ev 'this)], which doesn't @emph{look} like a +pair. But remember, @racket[asm-interp] is just giving us back +whatever is in the @racket[rax] register after calling this code: it's +giving us back @emph{bits}, not @emph{values}. But! You should +notice that these bits are encoding a pair value. If we look at the +three least significant bits, we see @binary[type-cons 3], aka +@racket[#,type-cons]: + +@ex[(eval:alts (bitwise-and #,(ev 'this) #,(binary ptr-mask)) + (bitwise-and this ptr-mask))] + +That tells us that @racket[#t] and @racket[#f] live at memory +addresses @racket[(bitwise-xor #,(ev 'this) #,(binary type-cons 3))] +and @racket[(+ (bitwise-xor #,(ev 'this) #,(binary type-cons 3)) 8)], +respectively. + +@margin-note{This is not actually true. Using Racket's +@racketmodname[ffi/unsafe] library provides a way to cast integers to +pointers and dereference arbitrary memory. In fact, +@racketmodname[ffi/unsafe] is the thing that makes @racket[asm-interp] +possible. The memory safety guarantee only applies to programs that +safely use @racketmodname[ffi/unsafe], which is easiest to do by not +using it all!} + +Now, how can we fetch them? On the Racket side of things, we just +have an integer and an integer is not a pointer in Racket. By design, +the language does not give you a way to cast an arbitrary integer to +some kind of pointer datatype that can be dereferenced. + This is important to +guarantee memory safety. Of course, @racket[asm-interp] offers a huge +back-door to that safety, so we can whip up our own operation to +dereference whatever memory address we'd like: + +@#reader scribble/comment-reader +(ex +;; Integer -> Integer +;; Fetch the word at given address +(define (mem-ref ptr) + (asm-interp + (prog (Global 'entry) + (Label 'entry) + (Mov r8 ptr) + (Mov rax (Mem r8)) + (Ret)))) + +(eval:alts (mem-ref (bitwise-xor #,(ev 'this) type-cons)) + (mem-ref (bitwise-xor this type-cons))) +(eval:alts (mem-ref (+ (bitwise-xor #,(ev 'this) type-cons) 8)) + (mem-ref (+ (bitwise-xor this type-cons) 8))) +) + +And while these also don't look like @racket[#t] and @racket[#f], +remember: + +@ex[(value->bits #t) + (value->bits #f)] + + +Aside: We should be very careful with this operation; you can do bad +things with it: + +@ex[(eval:alts (mem-ref #,(hex #xDEADBEEF)) + (eval:error (mem-ref #xDEADBEEF)))] + +We're now in a position to actually reconstruct the pair value on +the Racket side of things: + +@#reader scribble/comment-reader +(ex +;; Bits -> (cons Value Value) +;; Constructs a pair from bits encoding a pair value +(define (cons-bits->cons b) + (cons (bits->value (mem-ref (+ b (- 0 type-cons)))) + (bits->value (mem-ref (+ b (- 8 type-cons)))))) + +(eval:alts (cons-bits->cons #,(ev 'this)) + (cons-bits->cons this)) +) + + +And this works great so long as the values in the pair aren't +themselves tagged pointers, which of course, they could be! What +should we do in that case? Well, figure out what kind of pointer they +are, dereference their contents and construct the appropriate kind of +Racket value (either a box or a pair). We can do this recursively to +complete convert from whatever encoding of a value we get back into +the corresponding Racket value. In other words, extending +@racket[bits->value] to work in the presence of tagged pointer values +involves just the kind of thing we've written: + +@codeblock-include["hustle/types.rkt"] + +You'll notice that instead of the @racket[mem-ref] we wrote, it uses +Racket's own ``unsafe'' operations. The only difference is that this +is more efficient, bypassing the overhead of @racket[asm-interp]. + +With @racket[bits->value] in place, we can now build up some utilities +for running programs with the run-time system linked in and using +@racket[bits->value] to construct the result value: + +@codeblock-include["hustle/run.rkt"] + +Let's make the list @racket['(1 2 3)]. Remember that @racket['(1 2 +3)] is just @racket[(cons 1 (cons 2 (cons 3 '())))]. + +@#reader scribble/comment-reader +(ex +(run + (prog (Global 'entry) + (Label 'entry) + (Push rbx) + (Mov rbx rdi) + (Mov rax (value->bits 1)) + (Mov (Mem rbx 0) rax) + (Mov rax rbx) + (Add rax (+ 16 type-cons)) + (Mov (Mem rbx 8) rax) + (Mov rax (value->bits 2)) + (Mov (Mem rbx 16) rax) + (Mov rax rbx) + (Add rax (+ 32 type-cons)) + (Mov (Mem rbx 24) rax) + (Mov (Mem rbx 24) rax) + (Mov rax (value->bits 3)) + (Mov (Mem rbx 32) rax) + (Mov rax (value->bits '())) + (Mov (Mem rbx 40) rax) + (Mov rax rbx) + (Xor rax type-cons) + (Add rbx (* 8 6)) ; account for 6 words used + (Pop rbx) + (Ret)))) + + +These instructions create a list that is laid out in the heap like +this: + +@make-heap-diagram[ + '((cons 0) + 1 + (cons 2) + 2 + (cons 4) + 3 + '())] + +@margin-note{See if you can construct the list this way.} +Of course there are many ways to make the same list. We could, for +example, write instructions that made exactly the same list but +laid out like this: + +@make-heap-diagram[ + '((cons 4) + 3 + '() + 2 + (cons 0) + 1 + (cons 2))] + +Both of these would result in the same value from the perspective of +@racket[bits->value]. + +Now that we can make examples and have a good idea of how to write +instructions to create boxes and pairs in memory, let's write the +compiler. + + @section[#:tag-prefix prefix]{A Compiler for @this-lang} -The compiler for @this-lang is essentially the same as for Fraud, although -now with support for the new primitives: @racket[box], @racket[unbox], -@racket[box?], @racket[cons], @racket[car], @racket[car], -@racket[cdr], @racket[cons?], and @racket[empty?]: +There aren't any new expression forms in @this-lang; all of the work +is done in the implementation of the new primitives. Predicates like +@racket[box?], @racket[cons?], and @racket[empty?] are simple: +@racket[box?] and @racket[cons?] mask the pointer tag bits and compare +against the appropriate tag; @racket[empty?] tests whether the bits +are equal the bits for @racket['()]. + +For @racket[box], we know the argument to the @racket[box] constructor +will be in @racket[rax] register and we need to emit code that will: +write that value into memory at the current heap pointer location, +move and tag a pointer to that memory into @racket[rax], and finally +increment @racket[rbx] to account for the memory used: + +@ex[ +(compile-op1 'box) +(exec (parse '(box 10)))] -@codeblock-include["hustle/compile-ops.rkt"] -We can now confirm that the compiler generates code similar to what we -wrote by hand above: +This creates a box value in memory that looks like this: + +@make-heap-diagram['((box 0) 10)] + + +To @racket[unbox] a box value, again we have the argument in the +@racket[rax] register. We must check that the argument actually is a +box by checking its tag, signalling a run-time type error if its not. +If that succeeds, we can dereference the memory by reading the memory +location pointed to by the tagged pointer. When dereferencing, we +account for the tag by subtracting it as an offset. @ex[ -(define (show e c) - (compile-e (parse e) c)) +(compile-op1 'unbox) +(exec (parse '(unbox (box 10))))] -(show '(box 7) '()) -] +For @racket[cons], which is a binary operator, we know the first +argument will be the first element of the stack and that the second +argument will be in the @racket[rax] register. The compiler emits +code that pops the argument from the stack, writes both to memory, +creates a tagged pointer to the memory in @racket[rax], and increments +@racket[rbx] by @racket[16] to account for the two words of memory +used. + +@ex[ +(compile-op2 'cons) +(exec (parse '(cons 1 2)))] + +In order to avoid using a temporary register, this code writes the +@emph{second} argument first, but at offset @racket[8], then pops into +@racket[rax], writing the first argument second at offset @racket[0]. +It copies @racket[rbx] into @racket[rax], tags it a pair, then +increments @racket[rbx] appropriately. It creates a pair in memory +that looks like this: -This moves the encoding of @racket[7] into @racket['rax], then writes -it into the memory address pointed to by @racket['rbx], i.e. the next -free memory location. That address is then moved to @racket['rax] and -tagged as a box, which is the result of the expression. The final -step is to increment @racket['rbx] by @racket[8] to advance the free -memory pointer since one word of memory is now used. +@make-heap-diagram['((cons 0) 1 2)] -Suppose we have a box value bound to variable @racket[x], then this -code will unbox the value: +Accessing the parts of a pair is similar to @racket[unbox]: it checks +the type, then reads the memory address at the appropriate offset. @ex[ -(show '(unbox x) '(x)) -] +(compile-op1 'car) +(compile-op1 'cdr) +(exec (parse '(car (cons 1 2)))) +(exec (parse '(cdr (cons 1 2))))] -This loads @racket[x] from the stack into @racket['rax], then does tag -checking to make sure it's a box pointer, after which it erases the -tag to reveal the address and loads that memory address into -@racket['rax], thereby retrieving the value in the box. -The way that @racket[cons], @racket[car], and @racket[cdr] work are -essentially the same, except that pairs hold two values instead of -one: +Notice here that in @racket[car], the offset is @racket[#,(- 0 +type-cons)], which is @racket[(- 0 type-cons)], while in @racket[cdr] +it is @racket[#,(- 8 type-cons)], which is @racket[(- 8 type-cons)]. + +We now have all the pieces to make lists or nested lists. @ex[ -(show '(cons 7 5) '()) -(show '(car x) '(x)) -(show '(cdr x) '(x)) -] +(exec (parse '(cons 1 (cons 2 (cons 3 '())))))] -We can now see why we chose to layout pairs with the @racket[cdr] -first and @racket[car] second. Since @racket[cons] is a binary -operation, the expression which produces the @racket[car] value will -be evaluated first and pushed on the stack. Then the expression that -produces the @racket[cdr] value will execute with its result sitting -in @racket[rax]. So at this point it's easiest to write out the -@racket[cdr] since it's already sitting in a register. Once we do -that, we can pop the @racket[car] value into @racket['rax] and write -that. Hence our choice for the layout. -@section[#:tag "hustle-run-time"]{A Run-Time for @this-lang} -First, we extend our runtime system's view of values to include -pointers and use C @tt{struct} to represent them: +Putting it all together we get the compiler for the new primitives: +@racket[box], @racket[unbox], @racket[box?], @racket[cons], +@racket[car], @racket[car], @racket[cdr], @racket[cons?], and +@racket[empty?]: -@filebox-include[fancy-c hustle "values.h"] +@codeblock-include["hustle/compile-ops.rkt"] -The implementation of @tt{val_typeof} is extended to handle -pointer types: -@filebox-include[fancy-c hustle "values.c"] -The rest of the run-time system for @this-lang is more involved for two -main reasons: - -The first is that the compiler relies on a pointer to free memory -residing in @racket['rbx]. The run-time system will be responsible -for allocating this memory and initializing the @racket['rdi] -register. To allocate memory, it uses @tt{malloc}. It passes the -pointer returned by @tt{malloc} to the @tt{entry} function. The -protocol for calling functions in C says that the first argument will -be passed in the @racket['rdi] register. Since @tt{malloc} produces -16-byte aligned addresses on 64-bit machines, @racket['rdi] is + +@section[#:tag "hustle-run-time"]{A Run-Time for @this-lang} + + +Our compiler relies on the fact that @racket[rbx] points to available +memory, but where did this memory come from? Well that will be the +job of the run-time system: before it runs the code our compiler +generated, it will ask the operating system to allocate a block of +memory and then pass its address as an argument to the @racket[entry] +function the compiler emits. + + +To allocate memory, it uses @tt{malloc}. It passes the pointer +returned by @tt{malloc} to the @tt{entry} function. The protocol for +calling functions in the System V ABI says that the first argument +will be passed in the @racket[rdi] register. Since @tt{malloc} +produces 16-byte aligned addresses on 64-bit machines, @racket[rdi] is initialized with an address that ends in @code[#:lang "racket"]{#b000}, satisfying our assumption about addresses. Once the runtime system has provided the heap address in -@racket['rdi], it becomes our responsibility to keep track of that -value. Because @racket['rdi] is used to pass arguments to C functions, -we can't keep our heap pointer in @racket['rdi] and expect it to be +@racket[rdi], it becomes our responsibility to keep track of that +value. Because @racket[rdi] is used to pass arguments to C functions, +we can't keep our heap pointer in @racket[rdi] and expect it to be saved. This leaves us with two options: @itemlist[ - @item{We can ensure that we save @racket['rdi] somewhere safe whenever we - might call a C function} + @item{We can ensure that we save @racket[rdi] somewhere safe whenever we + might call an external function} - @item{We can move the value away from @racket['rdi] as soon as possible and - never have to worry about @racket['rdi] being clobbered during a call + @item{We can move the value away from @racket[rdi] as soon as possible and + never have to worry about @racket[rdi] being clobbered during a call to a C function (as long as we pick a good place!)} ] -We've decided to use the second option, which leaves the choice of @emph{where} -to move the value once we receive it from the runtime system. As usual, we will -consult the System V Calling Convention, which tells us that @racket['rbx] is a -@emph{callee save} register, which means that any C function we might call is -responsible for ensuring that the value in the register is saved and restored. -In other words: we, the caller, don't have to worry about it! Because of this -we're going to use @racket['rbx] to store our heap pointer. You can see -that we do this in the compiler with @racket[(Mov 'rbx 'rdi)] as part -of our entry code. +We decided to use the second option, which leaves the choice of +@emph{where} to move the value once we receive it from the runtime +system. As usual, we will consult the System V Calling Convention, +which tells us that @racket[rbx] is a @emph{callee save} register, +which means that any external function we might call is responsible +for ensuring that the value in the register is saved and restored. In +other words: we, the caller, don't have to worry about it! Because of +this we're going to use @racket[rbx] to store our heap pointer. You +can see that we do this in the compiler with @racket[(Mov rbx rdi)] as +part of our entry code. @filebox-include[fancy-c hustle "main.c"] -The second complication comes from printing. Now that values include -inductively defined data, the printer must recursively traverse these -values to print them. It also must account for the wrinkle of how the + +@subsection{Updating the run-time system's notion of Values} + +We extend our runtime system's view of values to include +pointers and use C @tt{struct} to represent them: + +@filebox-include[fancy-c hustle "values.h"] + +The implementation of @tt{val_typeof} is extended to handle +pointer types: + +@filebox-include[fancy-c hustle "values.c"] + + +@subsection{Printing Values} + +Now that values include inductively defined data, the printer must +recursively traverse these values to print them (this is exactly +analogous to how @racket[bits->value] had to recursively construct +values, too). It also must account for the wrinkle of how the printing of proper and improper lists is different: @filebox-include[fancy-c hustle "print.c"] @section[#:tag-prefix prefix]{Correctness} + The statement of correctness for the @|this-lang| compiler is the same -as the previous one: +as the previous one, although it is worth noting that it's use of +@racket[bits->value] within @racket[exec/io] is hiding some subtleties +since it recursively constructs the result value. + +@margin-note{FIXME: should this be defined in terms of Answers?} @bold{Compiler Correctness}: @emph{For all @racket[e] @math{∈} -@tt{ClosedExpr}, @racket[i], @racket[o] @math{∈} @tt{String}, and @racket[v] -@math{∈} @tt{Value}, if @racket[(interp/io e i)] equals @racket[(cons -v o)], then @racket[(exec/io e i)] equals -@racket[(cons v o)].} +@tt{ClosedExpr}, @racket[i], @racket[o] @math{∈} @tt{String}, and @racket[a] +@math{∈} @tt{Answer}, if @racket[(interp/io e i)] equals @racket[(cons +a o)], then @racket[(exec/io e i)] equals +@racket[(cons a o)].} diff --git a/www/notes/knock.scrbl b/www/notes/knock.scrbl index efd517f3..1f3f904f 100644 --- a/www/notes/knock.scrbl +++ b/www/notes/knock.scrbl @@ -325,7 +325,7 @@ It's fairly straightforward: [((cons p ps) (cons e es)) (match (interp-match-pat p v r) [#f (interp-match v ps es r ds)] - [r (interp-env e r ds)])])) + [r (interp-e e r ds)])])) ) The complete interpreter: @@ -337,7 +337,7 @@ We can now see it in action: @ex[ (define (run e) - (interp-env (parse-e e) '() '())) + (interp-e (parse-e e) '() '())) (run '(match 1 [1 #t] [_ #f])) (run '(match 2 [1 #t] [_ #f])) diff --git a/www/notes/loot.scrbl b/www/notes/loot.scrbl index 501e3176..2e55f894 100644 --- a/www/notes/loot.scrbl +++ b/www/notes/loot.scrbl @@ -187,7 +187,7 @@ forms are @racket[λ]s and applications: @#reader scribble/comment-reader (racketblock ;; Expr REnv Defns -> Answer -(define (interp-env e r ds) +(define (interp-e e r ds) (match e ;; ... [(Lam _ xs e) '...] @@ -236,16 +236,16 @@ in what we know so far: @#reader scribble/comment-reader (racketblock ;; Expr REnv Defns -> Answer -(define (interp-env e r ds) +(define (interp-e e r ds) (match e ;; ... [(Lam _ xs e) (λ ??? '...)] [(App e es) - (match (interp-env e r ds) + (match (interp-e e r ds) ['err 'err] [f - (match (interp-env* es r ds) + (match (interp-e* es r ds) ['err 'err] [vs (apply f vs)])])])) @@ -261,16 +261,16 @@ number of arguments: @#reader scribble/comment-reader (racketblock ;; Expr REnv Defns -> Answer -(define (interp-env e r ds) +(define (interp-e e r ds) (match e ;; ... [(Lam _ xs e) (λ vs '...)] [(App e es) - (match (interp-env e r ds) + (match (interp-e e r ds) ['err 'err] [f - (match (interp-env* es r ds) + (match (interp-e* es r ds) ['err 'err] [vs (apply f vs)])])])) @@ -284,16 +284,16 @@ Translating that to code, we get: @#reader scribble/comment-reader (racketblock ;; Expr REnv Defns -> Answer -(define (interp-env e r ds) +(define (interp-e e r ds) (match e ;; ... [(Lam _ xs e) - (λ vs (interp-env e (zip xs vs) ds))] + (λ vs (interp-e e (zip xs vs) ds))] [(App e es) - (match (interp-env e r ds) + (match (interp-e e r ds) ['err 'err] [f - (match (interp-env* es r ds) + (match (interp-e* es r ds) ['err 'err] [vs (apply f vs)])])])) @@ -334,16 +334,16 @@ in the (Racket) function: @#reader scribble/comment-reader (racketblock ;; Expr REnv Defns -> Answer -(define (interp-env e r ds) +(define (interp-e e r ds) (match e ;; ... [(Lam _ xs e) - (λ vs (interp-env e (append (zip xs vs) r)) ds)] + (λ vs (interp-e e (append (zip xs vs) r)) ds)] [(App e es) - (match (interp-env e r ds) + (match (interp-e e r ds) ['err 'err] [f - (match (interp-env* es r ds) + (match (interp-e* es r ds) ['err 'err] [vs (apply f vs)])])])) @@ -354,20 +354,20 @@ The last remaining issue is we should do some type and arity-checking: @#reader scribble/comment-reader (racketblock ;; Expr REnv Defns -> Answer -(define (interp-env e r ds) +(define (interp-e e r ds) (match e ;; ... [(Lam _ xs e) (λ vs ; check arity matches (if (= (length xs) (length vs)) - (interp-env e (append (zip xs vs) r) ds) + (interp-e e (append (zip xs vs) r) ds) 'err))] [(App e es) - (match (interp-env e r ds) + (match (interp-e e r ds) ['err 'err] [f - (match (interp-env* es r ds) + (match (interp-e* es r ds) ['err 'err] [vs (if (procedure? f) @@ -398,7 +398,7 @@ So for now we interpret variables as follows: (define (interp-var x r ds) (match (lookup r x) ['err (match (defns-lookup ds x) - [(Defn f xs e) (interp-env (Lam f xs e) '() ds)] + [(Defn f xs e) (interp-e (Lam f xs e) '() ds)] [#f 'err])] [v v])) ) @@ -476,7 +476,7 @@ We can also ``import'' Racket functions in to Loot: @ex[ -(interp-env (parse-e '(expt 2 10)) +(interp-e (parse-e '(expt 2 10)) (list (list 'expt expt)) '()) ] @@ -542,23 +542,23 @@ to be in the (Racket) function: @#reader scribble/comment-reader (racketblock ;; Expr REnv Defns -> Answer -(define (interp-env e r ds) +(define (interp-e e r ds) (match e ;;... [(Lam _ xs e) (Closure xs e r)] [(App e es) - (match (interp-env e r ds) + (match (interp-e e r ds) ['err 'err] [f - (match (interp-env* es r ds) + (match (interp-e* es r ds) ['err 'err] [vs (match f [(Closure xs e r) ; check arity matches (if (= (length xs) (length vs)) - (interp-env e (append (zip xs vs) r) ds) + (interp-e e (append (zip xs vs) r) ds) 'err)] [_ 'err])])])])) ) @@ -620,7 +620,7 @@ interpretation of functions: [(Closure xs e r) ; check arity matches (if (= (length xs) (length vs)) - (interp-env e (append (zip xs vs) r) '()) + (interp-e e (append (zip xs vs) r) '()) 'err)] [_ 'err]))