Skip to content

Commit b73a099

Browse files
yuweizzzyuweizzz
authored
feat: parse http2 data in text mode (#580)
* h2 demo * gzip * http2 request test * update test file * update test * update bin file
1 parent 7e1ad52 commit b73a099

File tree

10 files changed

+338
-4
lines changed

10 files changed

+338
-4
lines changed

‎pkg/event_processor/http2_request.go‎

Lines changed: 158 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,158 @@
1+
// Copyright 2024 yuweizzz <[email protected]>. All Rights Reserved.
2+
//
3+
// Licensed under the Apache License, Version 2.0 (the "License");
4+
// you may not use this file except in compliance with the License.
5+
// You may obtain a copy of the License at
6+
//
7+
// http://www.apache.org/licenses/LICENSE-2.0
8+
//
9+
// Unless required by applicable law or agreed to in writing, software
10+
// distributed under the License is distributed on an "AS IS" BASIS,
11+
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12+
// See the License for the specific language governing permissions and
13+
// limitations under the License.
14+
15+
package event_processor
16+
17+
import (
18+
"bufio"
19+
"bytes"
20+
"compress/gzip"
21+
"errors"
22+
"fmt"
23+
"io"
24+
"log"
25+
26+
"golang.org/x/net/http2"
27+
"golang.org/x/net/http2/hpack"
28+
)
29+
30+
const H2Magic = "PRI * HTTP/2.0\r\n\r\nSM\r\n\r\n"
31+
const H2MagicLen = len(H2Magic)
32+
33+
type HTTP2Request struct {
34+
framer *http2.Framer
35+
packerType PacketType
36+
isDone bool
37+
isInit bool
38+
reader *bytes.Buffer
39+
bufReader *bufio.Reader
40+
}
41+
42+
func (h2r *HTTP2Request) detect(payload []byte) error {
43+
data := string(payload[0:H2MagicLen])
44+
if data != H2Magic {
45+
return errors.New("Not match http2 magic")
46+
}
47+
return nil
48+
}
49+
50+
func (h2r *HTTP2Request) Init() {
51+
h2r.reader = bytes.NewBuffer(nil)
52+
h2r.bufReader = bufio.NewReader(h2r.reader)
53+
h2r.framer = http2.NewFramer(nil, h2r.bufReader)
54+
h2r.framer.ReadMetaHeaders = hpack.NewDecoder(0, nil)
55+
}
56+
57+
func (h2r *HTTP2Request) Write(b []byte) (int, error) {
58+
if !h2r.isInit {
59+
h2r.Init()
60+
h2r.isInit = true
61+
}
62+
length, err := h2r.reader.Write(b)
63+
if err != nil {
64+
return 0, err
65+
}
66+
return length, nil
67+
}
68+
69+
func (h2r *HTTP2Request) ParserType() ParserType {
70+
return ParserTypeHttp2Request
71+
}
72+
73+
func (h2r *HTTP2Request) PacketType() PacketType {
74+
return h2r.packerType
75+
}
76+
77+
func (h2r *HTTP2Request) Name() string {
78+
return "HTTP2Request"
79+
}
80+
81+
func (h2r *HTTP2Request) IsDone() bool {
82+
return h2r.isDone
83+
}
84+
85+
func (h2r *HTTP2Request) Display() []byte {
86+
_, err := h2r.bufReader.Discard(H2MagicLen)
87+
if err != nil {
88+
log.Println("[http2 request] Discard HTTP2 Magic error:", err)
89+
return h2r.reader.Bytes()
90+
}
91+
var encoding string
92+
dataBuf := bytes.NewBuffer(nil)
93+
frameBuf := bytes.NewBufferString("")
94+
for {
95+
f, err := h2r.framer.ReadFrame()
96+
if err != nil {
97+
if err != io.EOF {
98+
log.Println("[http2 request] Dump HTTP2 Frame error:", err)
99+
}
100+
break
101+
}
102+
switch f := f.(type) {
103+
case *http2.MetaHeadersFrame:
104+
frameBuf.WriteString(fmt.Sprintf("\nFrame Type\t=>\tHEADERS\n"))
105+
for _, header := range f.Fields {
106+
frameBuf.WriteString(fmt.Sprintf("%s\n", header.String()))
107+
if header.Name == "content-encoding" {
108+
encoding = header.Value
109+
}
110+
}
111+
case *http2.DataFrame:
112+
_, err := dataBuf.Write(f.Data())
113+
if err != nil {
114+
log.Println("[http2 request] Write HTTP2 Data Frame buffuer error:", err)
115+
}
116+
default:
117+
fh := f.Header()
118+
frameBuf.WriteString(fmt.Sprintf("\nFrame Type\t=>\t%s\n", fh.Type.String()))
119+
}
120+
}
121+
// merge data frame
122+
if dataBuf.Len() > 0 {
123+
frameBuf.WriteString(fmt.Sprintf("\nFrame Type\t=>\tDATA\n"))
124+
payload := dataBuf.Bytes()
125+
switch encoding {
126+
case "gzip":
127+
reader, err := gzip.NewReader(bytes.NewReader(payload))
128+
if err != nil {
129+
log.Println("[http2 request] Create gzip reader error:", err)
130+
break
131+
}
132+
payload, err = io.ReadAll(reader)
133+
if err != nil {
134+
log.Println("[http2 request] Uncompress gzip data error:", err)
135+
break
136+
}
137+
h2r.packerType = PacketTypeGzip
138+
defer reader.Close()
139+
default:
140+
h2r.packerType = PacketTypeNull
141+
}
142+
frameBuf.Write(payload)
143+
}
144+
return frameBuf.Bytes()
145+
}
146+
147+
func init() {
148+
h2r := &HTTP2Request{}
149+
h2r.Init()
150+
Register(h2r)
151+
}
152+
153+
func (h2r *HTTP2Request) Reset() {
154+
h2r.isDone = false
155+
h2r.isInit = false
156+
h2r.reader.Reset()
157+
h2r.bufReader.Reset(h2r.reader)
158+
}

‎pkg/event_processor/http2_responese.go‎

Lines changed: 153 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,153 @@
1+
// Copyright 2024 yuweizzz <[email protected]>. All Rights Reserved.
2+
//
3+
// Licensed under the Apache License, Version 2.0 (the "License");
4+
// you may not use this file except in compliance with the License.
5+
// You may obtain a copy of the License at
6+
//
7+
// http://www.apache.org/licenses/LICENSE-2.0
8+
//
9+
// Unless required by applicable law or agreed to in writing, software
10+
// distributed under the License is distributed on an "AS IS" BASIS,
11+
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12+
// See the License for the specific language governing permissions and
13+
// limitations under the License.
14+
15+
package event_processor
16+
17+
import (
18+
"bufio"
19+
"bytes"
20+
"compress/gzip"
21+
"fmt"
22+
"io"
23+
"log"
24+
25+
"golang.org/x/net/http2"
26+
"golang.org/x/net/http2/hpack"
27+
)
28+
29+
type HTTP2Response struct {
30+
framer *http2.Framer
31+
packerType PacketType
32+
isDone bool
33+
isInit bool
34+
reader *bytes.Buffer
35+
bufReader *bufio.Reader
36+
}
37+
38+
func (h2r *HTTP2Response) detect(payload []byte) error {
39+
rd := bytes.NewReader(payload)
40+
buf := bufio.NewReader(rd)
41+
framer := http2.NewFramer(nil, buf)
42+
framer.ReadMetaHeaders = hpack.NewDecoder(0, nil)
43+
_, err := framer.ReadFrame()
44+
if err != nil {
45+
return err
46+
}
47+
return nil
48+
}
49+
50+
func (h2r *HTTP2Response) Init() {
51+
h2r.reader = bytes.NewBuffer(nil)
52+
h2r.bufReader = bufio.NewReader(h2r.reader)
53+
h2r.framer = http2.NewFramer(nil, h2r.bufReader)
54+
h2r.framer.ReadMetaHeaders = hpack.NewDecoder(0, nil)
55+
}
56+
57+
func (h2r *HTTP2Response) Write(b []byte) (int, error) {
58+
if !h2r.isInit {
59+
h2r.Init()
60+
h2r.isInit = true
61+
}
62+
length, err := h2r.reader.Write(b)
63+
if err != nil {
64+
return 0, err
65+
}
66+
return length, nil
67+
}
68+
69+
func (h2r *HTTP2Response) ParserType() ParserType {
70+
return ParserTypeHttp2Response
71+
}
72+
73+
func (h2r *HTTP2Response) PacketType() PacketType {
74+
return h2r.packerType
75+
}
76+
77+
func (h2r *HTTP2Response) Name() string {
78+
return "HTTP2Response"
79+
}
80+
81+
func (h2r *HTTP2Response) IsDone() bool {
82+
return h2r.isDone
83+
}
84+
85+
func (h2r *HTTP2Response) Display() []byte {
86+
var encoding string
87+
dataBuf := bytes.NewBuffer(nil)
88+
frameBuf := bytes.NewBufferString("")
89+
for {
90+
f, err := h2r.framer.ReadFrame()
91+
if err != nil {
92+
if err != io.EOF {
93+
log.Println("[http2 response] Dump HTTP2 Frame error:", err)
94+
}
95+
break
96+
}
97+
switch f := f.(type) {
98+
case *http2.MetaHeadersFrame:
99+
frameBuf.WriteString(fmt.Sprintf("\nFrame Type\t=>\tHEADERS\n"))
100+
for _, header := range f.Fields {
101+
frameBuf.WriteString(fmt.Sprintf("%s\n", header.String()))
102+
if header.Name == "content-encoding" {
103+
encoding = header.Value
104+
}
105+
}
106+
case *http2.DataFrame:
107+
_, err := dataBuf.Write(f.Data())
108+
if err != nil {
109+
log.Println("[http2 response] Write HTTP2 Data Frame buffuer error:", err)
110+
}
111+
default:
112+
fh := f.Header()
113+
frameBuf.WriteString(fmt.Sprintf("\nFrame Type\t=>\t%s\n", fh.Type.String()))
114+
}
115+
}
116+
// merge data frame
117+
if dataBuf.Len() > 0 {
118+
frameBuf.WriteString(fmt.Sprintf("\nFrame Type\t=>\tDATA\n"))
119+
payload := dataBuf.Bytes()
120+
switch encoding {
121+
case "gzip":
122+
reader, err := gzip.NewReader(bytes.NewReader(payload))
123+
if err != nil {
124+
log.Println("[http2 response] Create gzip reader error:", err)
125+
break
126+
}
127+
payload, err = io.ReadAll(reader)
128+
if err != nil {
129+
log.Println("[http2 response] Uncompress gzip data error:", err)
130+
break
131+
}
132+
h2r.packerType = PacketTypeGzip
133+
defer reader.Close()
134+
default:
135+
h2r.packerType = PacketTypeNull
136+
}
137+
frameBuf.Write(payload)
138+
}
139+
return frameBuf.Bytes()
140+
}
141+
142+
func init() {
143+
h2r := &HTTP2Response{}
144+
h2r.Init()
145+
Register(h2r)
146+
}
147+
148+
func (h2r *HTTP2Response) Reset() {
149+
h2r.isDone = false
150+
h2r.isInit = false
151+
h2r.reader.Reset()
152+
h2r.bufReader.Reset(h2r.reader)
153+
}

‎pkg/event_processor/iparser.go‎

Lines changed: 2 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -94,11 +94,9 @@ func NewParser(payload []byte) IParser {
9494
case ParserTypeHttpResponse:
9595
newParser = new(HTTPResponse)
9696
case ParserTypeHttp2Request:
97-
// TODO support HTTP2 request
98-
// via golang.org/x/net/http2
99-
//hpack.NewEncoder(buf)
97+
newParser = new(HTTP2Request)
10098
case ParserTypeHttp2Response:
101-
// TODO support HTTP2 response
99+
newParser = new(HTTP2Response)
102100
default:
103101
newParser = new(DefaultParser)
104102
}

‎pkg/event_processor/processor_test.go‎

Lines changed: 20 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -84,15 +84,35 @@ func TestEventProcessor_Serve(t *testing.T) {
8484

8585
lines = strings.Split(string(bufString), "\n")
8686
ok := true
87+
h2Req := 0
88+
h2Resq := 0
8789
for _, line := range lines {
8890
if strings.Contains(strings.ToLower(line), "dump") {
8991
t.Log(line)
9092
ok = false
9193
}
94+
// http2 parse error log
95+
if strings.Contains(line, "[http2 re") {
96+
t.Log(line)
97+
ok = false
98+
}
99+
// http2 parse count
100+
if strings.Contains(line, "Name:HTTP2Response") {
101+
h2Resq += 1
102+
}
103+
if strings.Contains(line, "Name:HTTP2Request") {
104+
h2Req += 1
105+
}
92106
}
93107
if err != nil {
94108
t.Fatalf("close error: %s", err.Error())
95109
}
110+
if h2Resq != 3 {
111+
t.Fatalf("some errors occurred: HTTP2Response lack")
112+
}
113+
if h2Req != 2 {
114+
t.Fatalf("some errors occurred: HTTP2Request lack")
115+
}
96116
if !ok {
97117
t.Fatalf("some errors occurred")
98118
}

‎pkg/event_processor/testdata/952293616935736.bin‎

110 Bytes
Binary file not shown.

‎pkg/event_processor/testdata/952293616935737.bin‎

406 Bytes
Binary file not shown.

‎pkg/event_processor/testdata/952293616935738.bin‎

193 Bytes
Binary file not shown.

‎pkg/event_processor/testdata/952293616935739.bin‎

545 Bytes
Binary file not shown.

‎pkg/event_processor/testdata/952293616935740.bin‎

384 Bytes
Binary file not shown.

‎pkg/event_processor/testdata/all.json‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -13,3 +13,8 @@
1313
{"DataType":1,"Timestamp":952293599178141,"Pid":469960,"Tid":469960,"DataLen":77,"Comm":[99,117,114,108,0,0,0,0,0,0,0,0,0,0,0,0],"Fd":5,"Version":771}
1414
{"DataType":0,"Timestamp":952293616935735,"Pid":469960,"Tid":469960,"DataLen":1179,"Comm":[99,117,114,108,0,0,0,0,0,0,0,0,0,0,0,0],"Fd":5,"Version":771}
1515
{"DataType":0,"Timestamp":952293617095621,"Pid":469960,"Tid":469960,"DataLen":1664,"Comm":[99,117,114,108,0,0,0,0,0,0,0,0,0,0,0,0],"Fd":5,"Version":771}
16+
{"DataType":1,"Timestamp":952293616935736,"Pid":469961,"Tid":469961,"DataLen":110,"Comm":[99,117,114,108,0,0,0,0,0,0,0,0,0,0,0,0],"Fd":5,"Version":771}
17+
{"DataType":0,"Timestamp":952293616935737,"Pid":469963,"Tid":469963,"DataLen":406,"Comm":[99,117,114,108,0,0,0,0,0,0,0,0,0,0,0,0],"Fd":5,"Version":771}
18+
{"DataType":1,"Timestamp":952293616935738,"Pid":469962,"Tid":469962,"DataLen":193,"Comm":[99,117,114,108,0,0,0,0,0,0,0,0,0,0,0,0],"Fd":5,"Version":771}
19+
{"DataType":0,"Timestamp":952293616935739,"Pid":469964,"Tid":469964,"DataLen":545,"Comm":[99,117,114,108,0,0,0,0,0,0,0,0,0,0,0,0],"Fd":5,"Version":771}
20+
{"DataType":0,"Timestamp":952293616935740,"Pid":469965,"Tid":469965,"DataLen":384,"Comm":[99,117,114,108,0,0,0,0,0,0,0,0,0,0,0,0],"Fd":5,"Version":771}

0 commit comments

Comments
 (0)