connection: keep-alive

Instead of requesting the connection: close - which it appears some
ESPs will close the connection during buffering the uart data, now
we request with connection: keep-alive and manually close the TCP
connection when we're done.

This is achieved by parsing the http headers, looking for content-length
converting the value to a 32bit value, then subtracting the chunks we
get from the uart until the length reaches zero.

Chunked encoding is still going to be a problem because the bytes
include "metadata" that says how much data we've got in a chunk
so the length won't match the number of bytes coming down, but chunked
encoding wasn't supported yet anyway, so it's still an open issue.

This has been tested on AT versions:

- 1.1.0.0
- 1.2.0.0
- 1.3.0.0
- 1.6.0.0 (which had the problem in the first place)

example/capture-esp.bas.txt

@@ -2,22 +2,25 @@
 1 RUN AT 3
 10 LAYER 0
 20 BANK 20 ERASE
-30 ON ERROR GO SUB 9000
 40 CLS : PRINT INVERSE 1;"HTTP 4k debug"; INVERSE 0
-145 t$="4k":u$="/10":%r=1: GO SUB 1000
+145 t$="4k":u$="/10":%r=1: GO SUB 1010
 900 ON ERROR
 910 PRINT FLASH 1;"All OK"
 920 STOP
 1010 PRINT AT %r,0;t$+" test..."
 1020 t$=t$+".bin"
-1030 IF %e=1 THEN GO TO 7000: ; banking
-1040 REM file test logic
+1030 IF %e=1 THEN GO TO 8000: ; banking
+1040 ON ERROR GO TO 1300
 1200 ../http-debug.dot -h data.remysharp.com -u u$ -f t$ -v 6
+1210 GO TO 5090
+1300 ON ERROR
+1310 PRINT AT %r,11; INVERSE 1;"fail"; INVERSE 0;". Verifying: "
+1320 GO TO 5110
 5090 PRINT AT %r,11;"done. Verifying: "
-5100 c$=t$+" -1 -mb 20"
-5110 .$ extract c$
-5120 %i=% BANK 20 PEEK 0
-5130 SAVE "4k-esp-bank.bin" BANK 20,%$2000,%$2000
+5110 SAVE "4k-esp-bank.bin" BANK 20,%$2000,%$2000
+5115 c$=t$+" -1 -mb 20"
+5120 .$ extract c$
+5130 %i=% BANK 20 PEEK 0
 8000 IF %i=$FF THEN PRINT AT %r,27; INVERSE 1;"OK"; INVERSE 0: ELSE PRINT AT %r,27; FLASH 1;"BAD": PAUSE 0: STOP
 8010 RETURN
 9000 ON ERROR

src/headers.asm

@@ -1,25 +1,237 @@
+	; DEFINE TEST_HEADERS
+	IFDEF TEST_HEADERS
+		OPT reset --zxnext --syntax=abfw
+		INCLUDE "constants.inc.asm"
+		DEVICE ZXSPECTRUM48
+		SLDOPT COMMENT WPMEM, LOGPOINT, ASSERTION
+
+		ORG $8000
+start:
+		di
+		exx
+		ld hl, s2
+		exx
+		call Headers.findContentLength
+
+		ld de, 1299
+		call Headers.contentLengthSub
+
+		; call Uart.read
+
+		jr $
+
+		INCLUDE "utils.asm"
+
+	MODULE Uart
+; Fake Uart.read
+;
+; A <- result
+; Modifies: BC
+read:
+		exx
+		ld a, (hl)
+		inc hl
+		exx
+		ret
+	ENDMODULE
+
+
+result:
+		DS 256
+	ENDIF
+
+
 	MODULE Headers
+
+	DEFB $AA
+contentLength:						; 32bit result
+	DISPLAY "Header.contentLength @ ",/H,$
+	DWORD 0
+
+	DEFB $AA
+buffer:
+	DISPLAY "Header.buffer @ ",/H,$
+	BLOCK 9, 0
+
+; Subtracts DE from contentLength
+;
+; DE=integer
+; Fz=all bytes consumed
+; Fc=error
+; Modifies: AF, HL, DE
+contentLengthSub:
+		or a					; reset carry for sbc
+		ld hl, (contentLength)		; do the LSW first
+		sbc hl, de
+		ld (contentLength), hl
+		ret nc
+
+		or a
+		ld hl, (contentLength+2)
+		ld de, 1
+		sbc hl, de
+		ld (contentLength+2), hl
+		jr z, .checkZero
+		ret
+.checkZero
+		ld hl, (contentLength)
+		ld a, h
+		or l
+		ret
+
+
+; Takes DE and searches the text for the content length setting it in (HL)
+; searching through the headers for `content-length: nnnn`. If successful
+; carry is clear and the Header.contentLength is set to 32bits.
+; This reduces DE as it works through the Uart.read
+;
+; DE=length of UART buffer
+; Fc=failed to find header
+; Modifies: AF, BC, DE, HL, IX
+findContentLength
+		ld hl, buffer
+
+		;; process a single header
+.processHeader
+		call Uart.read				; load A with the next character
+		dec de
+
+		;; convert character to uppercase
+		cp 'a'					; if A < 'a' then skip case shift
+		jr c, .noCaseShiftC
+		sub $20
+.noCaseShiftC
+
+		;; narrow down what we're looking for, ref:
+		;; https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers
+		cp 'C'
+		jr nz, .TestForLineEndingHeader
+
+		;; slurp to the `-` character
+.findDash
+		call Uart.read
+		dec de
+		cp CR					; always check if we have an EOL
+		jr z, .processHeader
+
+		cp '-'					; now check for the char we want
+		jr nz, .findDash
+
+		call Uart.read				; after `-` we're looking for L
+		dec de
+
+		;; shift case
+		cp 'a'					; if A < 'a' then skip case shift
+		jr c, .noCaseShiftL
+		sub $20
+.noCaseShiftL
+		cp CR
+		jr z, .processHeader			; is this EOL?
+
+		cp 'L'					; L?
+		jr nz, .slurpToEndOfHeader
+
+		call Uart.read				; E?
+		dec de
+		cp CR					; EOL?
+		jr z, .processHeader
+
+		cp 'a'					; if A < 'a' then skip case shift
+		jr c, .noCaseShiftE
+		sub $20
+.noCaseShiftE
+
+		cp 'E'					; this now the content-length
+		jr nz, .slurpToEndOfHeader
+
+.findColon
+		call Uart.read
+		dec de
+		cp CR					; always check if we have an EOL
+		jr z, .processHeader
+
+		cp ':'
+		jr nz, .findColon
+
+.eatSpaces
+		call Uart.read
+		dec de
+		cp CR					; always check if we have an EOL
+		jr z, .processHeader
+
+		cp ' '
+		jr z, .eatSpaces
+
+		;; now we have our numbers - store this in a buffer for later conversion
+		ld (hl), a
+		inc hl
+.captureNumeric
+		call Uart.read
+		dec de
+		cp CR					; always check if we have an EOL
+		jr z, .convertToUint32
+
+		ld (hl), a
+		inc hl
+		jr .captureNumeric
+
+.convertToUint32
+		push de
+		ld de, buffer
+		call atoui32
+		; CSP_BREAK
+		ld (contentLength), ix
+		ld (contentLength+2), hl
+		pop de
+		jp .slurpToEndOfAllHeaders
+
+.TestForLineEndingHeader
+		;; this could actually be a blank line in which case we need to return
+		cp CR
+		jr nz, .slurpToEndOfHeader
+		;; this was a CR and we've not found the content length
+
+		;; flush the next character: LF
+		call Uart.read
+		dec de
+		scf
+		ret
+
+.slurpToEndOfHeader
+		call Uart.read : dec de : cp CR : jr nz, .slurpToEndOfHeader
+		call Uart.read : dec de ; LR
+		jp .processHeader
+
+.slurpToEndOfAllHeaders
+		call Uart.read : dec de : cp CR : jr nz, .slurpToEndOfAllHeaders
+		call Uart.read : dec de ; LR
+		call Uart.read : dec de : cp CR : jr nz, .slurpToEndOfAllHeaders
+		call Uart.read : dec de ; LR
+		ret
+Parse
+
+
 Post
-		ld hl, Strings.post
+		ld hl, HeaderStrings.post
 		ld bc, 5
 		jr method
 
 Get
-		ld hl, Strings.get
+		ld hl, HeaderStrings.get
 		ld bc, 4
 method
 		ldir
 		ret
 
 GetTrailer
-		ld hl, Strings.reqTail
-		ld bc, Strings.reqTailLen
+		ld hl, HeaderStrings.reqTail
+		ld bc, HeaderStrings.reqTailLen
 		ldir
 		ret
 
 PostTrailer
-		ld hl, Strings.reqTail
-		ld bc, Strings.postLen
+		ld hl, HeaderStrings.reqTail
+		ld bc, HeaderStrings.postLen
 		ldir
 		ret
 
@@ -29,7 +241,7 @@ PostTrailer
 ; Modifies: AF
 Host
 		push hl
-		ld hl, Strings.host
+		ld hl, HeaderStrings.host
 		ld bc, 6
 		ldir
 		pop hl
@@ -50,22 +262,73 @@ copyHLtoDE
 		jr copyHLtoDE
 
 NewLine
-		ld hl, Strings.newLine
+		ld hl, HeaderStrings.newLine
 		ld bc, 2
 		ldir
 		ret
 
 EndPost
-		ld hl, Strings.emptyLine
+		ld hl, HeaderStrings.emptyLine
 		ld bc, 5
 		ldir
 		ret
 
 EndGet
-		ld hl, Strings.newLine
+		ld hl, HeaderStrings.newLine
 		ld bc, 3
 		ldir
 		ret
 
 
 	ENDMODULE
+
+	MODULE HeaderStrings
+emptyLine	DEFB CR, LF
+newLine		DEFB CR, LF, 0
+get		DEFB "GET "
+post		DEFB "POST "
+host		DEFB "Host: "
+reqTail		DEFB " HTTP/1.1", CR, LF, "Connection: keep-alive", CR, LF
+reqTailLen	EQU $-reqTail
+postLength	DEFB "Content-Type: application/x-www-form-urlencoded", CR, LF, "Content-Length: "
+postLen		EQU $-reqTail
+
+	ENDMODULE
+
+
+	IFDEF TEST_HEADERS
+
+s1:		DEFB "HTTP/1.1 200 OK",CR,LF
+		DEFB "Server: nginx/1.14.2",CR,LF
+		DEFB "Date: Fri, 23 Apr 2021 18:42:46 GMT",CR,LF
+		DEFB "Content-Type: text/html",CR,LF
+		DEFB "Content-Length: 6608",CR,LF
+		DEFB "Last-Modified: Fri, 10 Jul 2020 15:43:55 GMT",CR,LF
+		DEFB "Connection: keep-alive",CR,LF
+		DEFB "Permissions-Policy: interest-cohort=()",CR,LF
+		DEFB "Referrer-Policy: no-referrer",CR,LF
+		DEFB "Accept-Ranges: bytes",CR,LF
+		DEFB CR,LF
+		DEFB "<!DOCTYPE html>",0
+
+s2:
+		DEFB "HTTP/1.1 200 OK",CR,LF
+		DEFB "Content-Type: application/octet-stream",CR,LF
+		DEFB "Content-Length: 65536",CR,LF
+		DEFB "Date: Fri, 23 Apr 2021 22:22:26 GMT",CR,LF
+		DEFB "Connection: keep-alive",CR,LF
+		DEFB "Keep-Alive: timeout=5",CR,LF
+		DEFB CR,LF
+		DEFB "<!DOCTYPE html>",0
+s3:
+		DEFB "HTTP/1.1 200 OK",CR,LF
+		DEFB "Content-Type: application/octet-stream",CR,LF
+		DEFB "Date: Fri, 23 Apr 2021 22:22:26 GMT",CR,LF
+		DEFB "Connection: keep-alive",CR,LF
+		DEFB "Keep-Alive: timeout=5",CR,LF
+		DEFB CR,LF
+		DEFB "<!DOCTYPE html>",0
+
+
+		SAVESNA "headers.sna", start
+	ENDIF