用protobuf生成json结构插件实现

背景

json格式不便描述/统一管理

在日常中json数据格式应用场景很多。比如在restful请求/返回、业务通信协议、消息（nsq/kafka），广泛使用json。但是目前存在如下问题

• json数据协议分散在代码里，没有一个统一的描述方式
• json使用往往使用map结构，然后序列化。造成上下游使用混乱，容易出错
• 如果使用struct方式定义，那么需要手动写代码，很是繁琐
  比如以下示例：

json格式如下：
{
    "id": 123,
    "name": "fangming",
    "age": 18,
    "occ": {"id": 111, "industry": "professor"},
    "occs": [
        {"id": 111, "industry": "professor"},    
        {"id": 222, "industry": "engineer"}
    ],
}
为了描述他我们不得不定义一个结构，然后序列化
type User struct {
    Id   int64         `json:"id"`
    Name string        `json:"name"`
    Age  int32         `json:"age"`
    Occ  *Occupation   `json:"occ"`
    Occs []*Occupation `json:"occs"`
}
type Occupation struct {
    Id       int64  `json:"id"`
    Industry string `json:"industry"`
}

方案

用protobuf协议描述json

考虑到golang里可以直接根据struct序列化成json，我们很自然的想到的是通过生成struct结构来实现。想想常用的描述协议的2个格式(protobuf、thrift)，也可以用来生成struct结构。这里考虑到thrift改动太大，用protobuf描述更好些。

一个描述示例如下：

message User {
    required int64 id = 1;
    required string name = 2;
    required int32 age = 3;
    optional Occupation occ = 4;
    repeated Occupation occs = 5;
}
message Occupation {
    required int64 id = 1;
    required string industry = 2;
}

// 代表的json结构如下
{
    "id": 123,
    "name": "fangming",
    "age": 18,
    "occ": {"id": 111, "industry": "professor"},
    "occs": [
        {"id": 111, "industry": "professor"},
        {"id": 222, "industry": "engineer"}
    ],
}

实现方案

直接基于protoc支持的扩展功能，写扩展插件即可。这里开发自己的插件：gogofmqjson

实现细节

后续有时间再更新

git地址

https:// github.com/buptbill220/ protobuf/tree/master

支持特性

• json编码完整支持required、optional字段语义检测；required字段不填序列化/反序列化报错
• 比用protobuf本身生成更简洁的代码
• 生成支持json的marshal & unmarshal方法
• 支持Set方法（主要是解决成员变量为指针类型，需要手动使用&）
• Set方法支持builder模式

protobuf其他特性

• 支持Set方法（主要是解决成员变量为指针类型，需要手动使用&）
• Set方法支持builder模式

安装使用

json协议代码生成

• go get http:// github.com/buptbill220/ protobuf/protoc-gen-gogofmqjson
• protoc --gogofmqjson_out=./ xxx.proto

无缝迁移gogo

• go get http:// github.com/buptbill220/ protobuf/protoc-gen-gogo
• go get http:// github.com/buptbill220/ protobuf/protoc-gen-gogofast
• go get http:// github.com/buptbill220/ protobuf/protoc-gen-gofast

生成代码示例

// FmqJsonCode generated by protoc-gen-gogofmqjson. Do NOT EDIT
// Author: fangming
// Email: fangming@bytedance.com
// source: test.proto
// It is generated from these files:
// 	test.proto

// It has these top-level messages:
//== 	User
//== 	Occupation

package test

import (
	encoding_json "encoding/json"
	fmt "fmt"
)

type User struct {
	Id   *int64        `json:"id"`
	Name *string       `json:"name"`
	Age  *int32        `json:"age"`
	Occ  *Occupation   `json:"occ"`
	Occs []*Occupation `json:"occs"`
}

func (m *User) Reset()      { *m = User{} }
func (*User) ProtoMessage() {}

func (m *User) GetId() int64 {
	if m != nil && m.Id != nil {
		return *m.Id
	}
	return 0
}

func (m *User) SetId(v int64) *User {
	if m != nil {
		m.Id = &v
	}
	return m
}

func (m *User) GetName() string {
	if m != nil && m.Name != nil {
		return *m.Name
	}
	return ""
}

func (m *User) SetName(v string) *User {
	if m != nil {
		m.Name = &v
	}
	return m
}

func (m *User) GetAge() int32 {
	if m != nil && m.Age != nil {
		return *m.Age
	}
	return 0
}

func (m *User) SetAge(v int32) *User {
	if m != nil {
		m.Age = &v
	}
	return m
}

func (m *User) GetOcc() *Occupation {
	if m != nil {
		return m.Occ
	}
	return nil
}

func (m *User) SetOcc(v *Occupation) *User {
	if m != nil {
		m.Occ = v
	}
	return m
}

func (m *User) GetOccs() []*Occupation {
	if m != nil {
		return m.Occs
	}
	return nil
}

func (m *User) SetOccs(v []*Occupation) *User {
	if m != nil {
		m.Occs = v
	}
	return m
}

type Occupation struct {
	Id       *int64  `json:"id"`
	Industry *string `json:"industry"`
}

func (m *Occupation) Reset()      { *m = Occupation{} }
func (*Occupation) ProtoMessage() {}

func (m *Occupation) GetId() int64 {
	if m != nil && m.Id != nil {
		return *m.Id
	}
	return 0
}

func (m *Occupation) SetId(v int64) *Occupation {
	if m != nil {
		m.Id = &v
	}
	return m
}

func (m *Occupation) GetIndustry() string {
	if m != nil && m.Industry != nil {
		return *m.Industry
	}
	return ""
}

func (m *Occupation) SetIndustry(v string) *Occupation {
	if m != nil {
		m.Industry = &v
	}
	return m
}

func (m *User) Marshal() ([]byte, error) {
	if m == nil {
		return nil, fmt.Errorf("msg User is nil")
	}
	if err := m.Validate(); err != nil {
		return nil, err
	}
	return encoding_json.Marshal(m)
}

func (m *User) Unmarshal(data []byte) error {
	if m == nil {
		return fmt.Errorf("msg User is nil")
	}
	if err := encoding_json.Unmarshal(data, m); err != nil {
		return err
	}
	if err := m.Validate(); err != nil {
		return err
	}
	return nil
}

func (m *Occupation) Marshal() ([]byte, error) {
	if m == nil {
		return nil, fmt.Errorf("msg Occupation is nil")
	}
	if err := m.Validate(); err != nil {
		return nil, err
	}
	return encoding_json.Marshal(m)
}

func (m *Occupation) Unmarshal(data []byte) error {
	if m == nil {
		return fmt.Errorf("msg Occupation is nil")
	}
	if err := encoding_json.Unmarshal(data, m); err != nil {
		return err
	}
	if err := m.Validate(); err != nil {
		return err
	}
	return nil
}

func (m *User) Validate() error {
	if m == nil {
		return fmt.Errorf("msg User is nil")
	}
	if m.Id == nil {
		return fmt.Errorf("required field User.Id is nil")
	}
	if m.Name == nil {
		return fmt.Errorf("required field User.Name is nil")
	}
	if m.Age == nil {
		return fmt.Errorf("required field User.Age is nil")
	}
	if m.Occ != nil {
		if err := m.Occ.Validate(); err != nil {
			return err
		}
	}
	for _, p := range m.Occs {
		if err := p.Validate(); err != nil {
			return err
		}
	}
	return nil
}

func (m *Occupation) Validate() error {
	if m == nil {
		return fmt.Errorf("msg Occupation is nil")
	}
	if m.Id == nil {
		return fmt.Errorf("required field Occupation.Id is nil")
	}
	if m.Industry == nil {
		return fmt.Errorf("required field Occupation.Industry is nil")
	}
	return nil
}

测试使用/示例

func testUser() {
	user := &User{}
	data, err := user.Marshal()
	if err != nil {
		fmt.Printf("1: user marshal err %s\n", err.Error())
	}
	fmt.Printf("1: user marshal data %s\n", string(data))
	
	
	data, err = user.SetId(1).Marshal()
	if err != nil {
		fmt.Printf("2: user marshal err %s\n", err.Error())
	}
	fmt.Printf("2: user marshal data %s\n", string(data))
	
	data, err = user.SetAge(28).Marshal()
	if err != nil {
		fmt.Printf("3: user marshal err %s\n", err.Error())
	}
	fmt.Printf("3: user marshal data %s\n", string(data))
	
	data, err = user.SetName("fangming").Marshal()
	if err != nil {
		fmt.Printf("4: user marshal err %s\n", err.Error())
	}
	fmt.Printf("4: user marshal data %s\n", string(data))
	
	occ := &Occupation{}
	occ.SetId(1)
	data, err = user.SetOcc(occ).Marshal()
	if err != nil {
		fmt.Printf("5: user marshal err %s\n", err.Error())
	}
	fmt.Printf("5: user marshal data %s\n", string(data))
	
	occ.SetIndustry("computer")
	data, err = user.SetOcc(occ).Marshal()
	if err != nil {
		fmt.Printf("6: user marshal err %s\n", err.Error())
	}
	fmt.Printf("6: user marshal data %s\n", string(data))
	
	occs := []*Occupation{}
	occ = &Occupation{}
	occ.SetId(2)
	occs = append(occs, occ)
	data, err = user.SetOccs(occs).Marshal()
	if err != nil {
		fmt.Printf("7: user marshal err %s\n", err.Error())
	}
	fmt.Printf("7: user marshal data %s\n", string(data))
	
	occ.SetIndustry("teacher")
	occs = append(occs, occ)
	data, err = user.SetOccs(occs).Marshal()
	if err != nil {
		fmt.Printf("8: user marshal err %s\n", err.Error())
	}
	fmt.Printf("8: user marshal data %s\n", string(data))
}

/* 输出如下
1: user marshal err required field User.Id is nil
1: user marshal data
2: user marshal err required field User.Name is nil
2: user marshal data
3: user marshal err required field User.Name is nil
3: user marshal data
4: user marshal data {"id":1,"name":"fangming","age":28,"occ":null,"occs":null}
5: user marshal err required field Occupation.Industry is nil
5: user marshal data
6: user marshal data {"id":1,"name":"fangming","age":28,"occ":{"id":1,"industry":"computer"},"occs":null}
7: user marshal err required field Occupation.Industry is nil
7: user marshal data
8: user marshal data {"id":1,"name":"fangming","age":28,"occ":{"id":1,"industry":"computer"},"occs":[{"id":2,"industry":"teacher"},{"id":2,"industry":"teacher"}]}
*/