Add substitute next commands

7 files changed, 135 insertions, 17 deletions

diff --git a/json/read.go b/json/read.go
index f3a0a65..91589cf 100644
--- a/json/read.go
+++ b/json/read.go
@@ -29,7 +29,9 @@ const (
 
 type JSONReaderState int
 const (
+	// Immediately before a value
 	JSONReaderStateValue JSONReaderState = iota
+	// Immediate after a value
 	JSONReaderStateValueEnd
 )
 
diff --git a/main/command.go b/main/command.go
index 6d75974..04ac7f6 100644
--- a/main/command.go
+++ b/main/command.go
@@ -27,11 +27,18 @@ func (cmd PrintValueCommand) String() string {
 
 type NextCommand struct {}
 func (cmd NextCommand) exec(state *ProgramState) {
-	nextItem, err := state.in.Read()
+	nextItem, err := state.Read()
 	if err != nil {
 		panic("Missing next value")
 	}
+
+	state.prevStart = nextItem.PrevStart
+	state.start = nextItem.Start
+	state.end = nextItem.End
+	state.nextEnd = nextItem.NextEnd
+
 	state.value = []walk.Value{nextItem.Value}
+
 	state.pc++
 }
 func (cmd NextCommand) String() string {
@@ -40,7 +47,7 @@ func (cmd NextCommand) String() string {
 
 type AppendNextCommand struct {}
 func (cmd AppendNextCommand) exec(state *ProgramState) {
-	nextItem, err := state.in.Read()
+	nextItem, err := state.Read()
 	if err != nil {
 		panic("Missing next value")
 	}
@@ -58,9 +65,61 @@ func (cmd AppendNextCommand) String() string {
 	return "N"
 }
 
+type SubstituteNextCommand struct {
+	subex subex.Transducer
+}
+func (cmd SubstituteNextCommand) exec(state *ProgramState) {
+	item, err := state.Peek()
+	if err != nil {
+		panic("Missing next value")
+	}
+
+	newValue, notOk := runSubex(cmd.subex, []walk.Value{item.Value})
+
+	if notOk {
+		state.pc++
+	} else {
+		state.prevStart = item.PrevStart
+		state.start = item.Start
+		state.end = item.End
+		state.nextEnd = item.NextEnd
+		state.pc += 2
+		state.value = newValue
+	}
+}
+func (cmd SubstituteNextCommand) String() string {
+	return "n/.../"
+}
+
+type SubstituteAppendNextCommand struct {
+	subex subex.Transducer
+}
+func (cmd SubstituteAppendNextCommand) exec(state *ProgramState) {
+	item, err := state.Peek()
+	if err != nil {
+		panic("Missing next value")
+	}
+
+	newValue, notOk := runSubex(cmd.subex, []walk.Value{item.Value})
+
+	if notOk {
+		state.pc++
+	} else {
+		state.prevStart = item.PrevStart
+		state.start = item.Start
+		state.end = item.End
+		state.nextEnd = item.NextEnd
+		state.pc += 2
+		state.value = append(state.value, newValue...)
+	}
+}
+func (cmd SubstituteAppendNextCommand) String() string {
+	return "N/.../"
+}
+
 type MergeCommand struct {}
 func (cmd MergeCommand) exec(state *ProgramState) {
-	nextItem, err := state.in.Read()
+	nextItem, err := state.Read()
 	if err != nil {
 		panic("Missing next value")
 	}
@@ -85,6 +144,14 @@ func (cmd MergeCommand) String() string {
 	return "m"
 }
 
+type FullMergeCommand struct {}
+func (cmd FullMergeCommand) exec(state *ProgramState) {
+	panic("Unimplemented")
+}
+func (cmd FullMergeCommand) String() string {
+	return "M"
+}
+
 type DeleteValueCommand struct {}
 func (cmd DeleteValueCommand) exec(state *ProgramState) {
 	state.value = nil
diff --git a/main/lex.go b/main/lex.go
index a28975f..8e66890 100644
--- a/main/lex.go
+++ b/main/lex.go
@@ -183,7 +183,7 @@ func lexCommand(l *lexer) stateFunc {
 	case 's', 'S':
 		l.emit(TokenCommand)
 		return lexSubstitution
-	case 'x', 'X', 'y', 'Y', 'z', 'Z':
+	case 'x', 'X', 'y', 'Y', 'z', 'Z', 'n', 'N':
 		l.emit(TokenCommand)
 		if l.peek() == '/' {
 			return lexSubstitution
diff --git a/main/main.go b/main/main.go
index 3fb1cbf..47bcddb 100644
--- a/main/main.go
+++ b/main/main.go
@@ -11,11 +11,29 @@ import (
 type ProgramState struct {
 	value, xreg, yreg, zreg []walk.Value
 	start, prevStart, end, nextEnd bool
+	// TODO: This will only ever have 0 or 1 values, it is a slice out of laziness
+	peekStack []walk.WalkItem
 	in walk.StredReader
 	out walk.StredWriter
 	program []Command
 	pc int
 }
+func (state *ProgramState) Read() (walk.WalkItem, error) {
+	if len(state.peekStack) > 0 {
+		item := state.peekStack[len(state.peekStack) - 1]
+		state.peekStack = state.peekStack[:len(state.peekStack) - 1]
+		return item, nil
+	}
+	return state.in.Read()
+}
+func (state *ProgramState) Peek() (walk.WalkItem, error) {
+	item, err := state.Read()
+	if err != nil {
+		return walk.WalkItem{}, err
+	}
+	state.peekStack = append(state.peekStack, item)
+	return item, nil
+}
 
 type config struct {
 	quiet bool
@@ -38,7 +56,7 @@ func run(config config) {
 	}
 
 	for {
-		walkItem, err := state.in.Read()
+		walkItem, err := state.Read()
 		if err != nil {
 			break
 		}
diff --git a/main/main_test.go b/main/main_test.go
index 74f179b..be439a3 100644
--- a/main/main_test.go
+++ b/main/main_test.go
@@ -9,6 +9,7 @@ var miscInput string = `{"something":{"nested":"Here is my test value"},"array":
 
 func TestMain(t *testing.T) {
 	type test struct {
+		name string
 		program string
 		quiet bool
 		input string
@@ -17,62 +18,78 @@ func TestMain(t *testing.T) {
 
 	tests := []test {
 		{
+			name: "Verbose Extract",
 			program: `s/#(~(people)~$_@(1$_#(~(first_name)~$_.|(..$_){-0})-|(..$_){-0})-|(..$_){-0})-/p`,
 			quiet: true,
 			input: miscInput,
 			expected: `"Tom"`,
 		},
 		{
+			name: "Extract",
 			program: `s/#("people"$_ @(1 $_#("first_name"$_ .)-)-)-/p`,
 			quiet: true,
 			input: miscInput,
 			expected: `"Tom"`,
 		},
 		{
+			name: "Simple Extract",
 			program: "s/#(\"people\" @(1 #(\"first_name\" (.$a))-)-)-$_ `$a`/p",
 			quiet: true,
 			input: miscInput,
 			expected: `"Tom"`,
 		},
 		{
+			name: "Larger Extract",
 			program: "s/#(\"people\" @(2 (.$a))-)-$_ `$a`/p",
 			quiet: true,
 			input: miscInput,
 			expected: `{"first_name":"Charlie","last_name":"Chaplin","age":122}`,
 		},
 		{
+			name: "Extract ages",
 			program: "s/#(\"people\"$_ :(#(\"age\"$_ .)-):)-/p",
 			quiet: true,
 			input: miscInput,
 			expected: `[22,18,122,48]`,
 		},
 		{
+			name: "Low memory count people",
 			program: "aX/#(\"people\" :(#()#):)#$_ `1`/o es/#()#/{ xs/.{-0}+/p }",
 			quiet: true,
 			input: miscInput,
 			expected: "4",
 		},
 		{
+			name: "Get full names",
 			program: "s/#(\"people\"$_ .)-/{ s/:():/p as/:(#()#):/{ xdx } s/:(#((\"first_name\" | \"last_name\") .)#)-/X es/@(.#()-)-/{ xs/(#(\"first_name\" \".{-0}$a\")# | #(\"last_name\" \".{-0}$b\")# | .){-0}$_ `\"$a $b\"`/Xxs/-(..)@/p } }",
 			quiet: true,
 			input: miscInput,
 			expected: `["Charlie Johnson","Tom Johnson","Charlie Chaplin","John Johnson"]`,
 		},
 		{
+			name: "Get full names 2",
 			program: "s/#(\"people\"$_ .)-/{ s/:():/p as/:(#()#):/{ xdx } X/:(#((\"first_name\" | \"last_name\") .)#)-/o es/@(.#()-)-/{ xX/(#(\"first_name\" \".{-0}$a\")# | #(\"last_name\" \".{-0}$b\")# | .){-0}$_ `\"$a $b\"`/xs/-(..)@/p } }",
 			quiet: true,
 			input: miscInput,
 			expected: `["Charlie Johnson","Tom Johnson","Charlie Chaplin","John Johnson"]`,
 		},
 		{
+			name: "Change full names in place",
 			program: "s/#(\"people\" @(. #(\"first_name\" .)#)@)#/{ ms/#(\"people\" @(. (#(\"first_name\" \".{-0}$a\" \"last_name\" \".{-0}$b\")#$_) `#(\"name\" \"$a $b\")#`)@)#/ }",
 			input: miscInput,
 			expected: `{"something":{"nested":"Here is my test value"},"array":["Hello","world","these","are","values"],"people":[{"name":"Charlie Johnson","age":22},{"name":"Tom Johnson","age":18},{"name":"Charlie Chaplin","age":122},{"name":"John Johnson","age":48}]}`,
 		},
+		{
+			name: "Get full names with substitute next command",
+			program: "s/#( \"people\"$_ :( #( \"first_name\"$_ . )- )- )-/{ N/#( \"people\"$_ :( #( \"last_name\"$_ . )- )- )-/{ s/-( -( ~(.{-0}` `)- ~(.{-0})- )~ ):/p }}",
+			quiet: true,
+			input: miscInput,
+			expected: `["Charlie Johnson","Tom Johnson","Charlie Chaplin","John Johnson"]`,
+		},
 	}
 
-	for i, test := range tests {
-		t.Logf("Running test: %d", i)
+	for _, test := range tests {
+		t.Logf("Running test: %s", test.name)
 
 		var output strings.Builder
 		run(config {
diff --git a/main/parse.go b/main/parse.go
index 3c24e6c..36917ac 100644
--- a/main/parse.go
+++ b/main/parse.go
@@ -44,13 +44,7 @@ func (p *parser) parseSubex() subex.SubexAST {
 	if subexProgramToken.typ != TokenSubex {
 		panic("Missing subex from substitution")
 	}
-	var subexProgram string
-	if delim.val == "=" || delim.val == "~" || delim.val == "\"" || delim.val == "`" || delim.val == "^" {
-		subexProgram = delim.val + subexProgramToken.val + delim.val
-	} else {
-		subexProgram = subexProgramToken.val
-	}
-	reader := subex.NewStringRuneReader(subexProgram)
+	reader := subex.NewStringRuneReader(subexProgramToken.val)
 	subexAST := subex.Parse(reader)
 	delim = p.next()
 	if delim.typ != TokenSubstituteDelimiter {
@@ -66,9 +60,21 @@ func (p *parser) parseBasicCommand(commands []Command, commandChar rune) []Comma
 		case 'd':
 			return append(commands, DeleteValueCommand{})
 		case 'n':
-			return append(commands, NextCommand{})
+			delim := p.peek()
+			if delim.typ != TokenSubstituteDelimiter {
+				return append(commands, NextCommand{})
+			}
+			ast := p.parseSubex()
+			subex := subex.CompileTransducer(ast)
+			return append(commands, SubstituteNextCommand {subex}, JumpCommand {len(commands) + 3})
 		case 'N':
-			return append(commands, AppendNextCommand{})
+			delim := p.peek()
+			if delim.typ != TokenSubstituteDelimiter {
+				return append(commands, AppendNextCommand{})
+			}
+			ast := p.parseSubex()
+			subex := subex.CompileTransducer(ast)
+			return append(commands, SubstituteAppendNextCommand {subex}, JumpCommand {len(commands) + 3})
 		case 'm':
 			return append(commands, MergeCommand{})
 		case 's':
diff --git a/subex/parse.go b/subex/parse.go
index d825f75..b6bf2f6 100644
--- a/subex/parse.go
+++ b/subex/parse.go
@@ -516,7 +516,14 @@ func parseSubex(l RuneReader, minPower int, inType Type) (lhs SubexAST, outType
 			lhs = SubexASTCopyNumber{}
 		case '`':
 			outType = inType
-			lhs = parseValueReplacement(l, '`')
+			switch inType {
+			case ValueType:
+				lhs = parseValueReplacement(l, '`')
+			case RuneType:
+				lhs = parseRuneReplacement(l, '`')
+			default:
+				panic("Invalid inType")
+			}
 		case ' ':
 			if inType == RuneType {
 				outType = RuneType
@@ -540,6 +547,7 @@ func parseSubex(l RuneReader, minPower int, inType Type) (lhs SubexAST, outType
 	loop: for {
 		if minPower <= 20 {
 			next, outType2 := parseSubex(l, 21, inType)
+			// TODO: next might legitimately be SubexASTEmpty, e.g. ``
 			if next != nil && (next != SubexASTEmpty{}) {
 				outType = resolveTypes(outType, outType2)
 				lhs = SubexASTConcat{lhs, next}