出于性能原因,我正在 Go 中的 Google AppEngine 项目中编写一个模块,但需要能够从数据存储中的某些实体中读取数据。我编写了Go代码以便能够读取我在Python中构建的实体,但是我收到以下错误:
datastore: flattening nested structs leads to a slice of slices: field "Messages"
Python 中的模型定义:
class ModelB(ndb.Model):
msg_id = ndb.StringProperty(indexed=False)
cat_ids = ndb.StringProperty(repeated=True, indexed=False)
list_ids = ndb.StringProperty(repeated=True, indexed=False)
default_list_id_index = ndb.IntegerProperty(indexed=False)
class ModelA(ndb.Model):
date_join = ndb.DateTimeProperty(auto_now_add=True)
name = ndb.StringProperty()
owner_salutation = ndb.StringProperty(indexed=False)
owner_email_address = ndb.StringProperty()
logo_url = ndb.StringProperty(indexed=False)
...
messages = ndb.LocalStructuredProperty(ModelB, name='bm', repeated=True)
在围棋中:
type ModelB struct {
MessageID string `datastore:"msg_id,noindex"`
CategoryIDs []string `datastore:"cat_ids,noindex"`
ListIDs []string `datastore:"list_ids,noindex"`
DefaultListIDIndex int `datastore:"default_list_id_index,noindex"`
}
type ModelA struct {
DateJoin time.Time `datastore:"date_join,"`
Name string `datastore:"name,"`
OwnerSalutation string `datastore:"owner_salutation,noindex"`
OwnerEmailAddress string `datastore:"owner_email_address,"`
LogoURL string `datastore:"logo_url,noindex"`
Messages []ModelB `datastore:"bm,"`
}
我在这里做错了什么吗?只是 Go 与 Python 模型定义之间的功能不兼容吗?
尝试解码模型 B
重新定义ModelA
,如下所示:
import pb "appengine_internal/datastore"
import proto "code.google.com/p/goprotobuf/proto"
type ModelA struct {
DateJoin time.Time `datastore:"date_join,"`
Name string `datastore:"name,"`
OwnerSalutation string `datastore:"owner_salutation,noindex"`
OwnerEmailAddress string `datastore:"owner_email_address,"`
LogoURL string `datastore:"logo_url,noindex"`
Messages []ModelB `datastore:"-"`
}
// Load is implemented for the PropertyLoaderSaver interface.
func (seller *ModelA) Load(c <-chan datastore.Property) error {
f := make(chan datastore.Property, 100)
for p := range c {
if p.Name == "bm" {
var val pb.EntityProto
err := proto.Unmarshal([]byte(p.Value.(string)), &val)
if err != nil {
return err
}
//TODO: Store result as a new ModelB
} else {
f <- p
}
}
close(f)
return datastore.LoadStruct(seller, f)
}
但是我收到以下错误: proto: required field "{Unknown}" not set
Go 数据存储包不支持这样的两层切片。你可以有[]ModelB
,只要ModelB
不包含任何切片。或者,您可以在 ModelA
中使用ModelB
,ModelB
其中可以包含切片。但是你不能同时拥有[]ModelB
,ModelB
都有切片。请参阅错误条件的代码。您的选择:
- 不要在 Go 中这样做
- 编写您自己的数据存储反序列化程序来处理这种情况 - 这可能很难
- 更改 Python 数据结构以满足 Go 要求并重写数据
如果你挖掘得足够多,你会找到答案:
首先,在 Python 中定义 LocalStructuredProperty 属性时,需要设置keep_keys=True
class ModelB(ndb.Model):
msg_id = ndb.StringProperty(indexed=False)
cat_ids = ndb.StringProperty(repeated=True, indexed=False)
list_ids = ndb.StringProperty(repeated=True, indexed=False)
default_list_id_index = ndb.IntegerProperty(indexed=False)
class ModelA(ndb.Model):
date_join = ndb.DateTimeProperty(auto_now_add=True)
name = ndb.StringProperty()
owner_salutation = ndb.StringProperty(indexed=False)
owner_email_address = ndb.StringProperty()
logo_url = ndb.StringProperty(indexed=False)
...
messages = ndb.LocalStructuredProperty(ModelB, name='bm', repeated=True, keep_keys=True)
在我的代码中进行简单的重新定义,并在我的实体上进行映射,对每个实体进行put()
,从而修复了表示形式。
然后在我的 Go 代码中:
type ModelB struct {
MessageID string `datastore:"msg_id,noindex"`
CategoryIDs []string `datastore:"cat_ids,noindex"`
ListIDs []string `datastore:"list_ids,noindex"`
DefaultListIDIndex int `datastore:"default_list_id_index,noindex"`
}
type ModelA struct {
DateJoin time.Time `datastore:"date_join,"`
Name string `datastore:"name,"`
OwnerSalutation string `datastore:"owner_salutation,noindex"`
OwnerEmailAddress string `datastore:"owner_email_address,"`
LogoURL string `datastore:"logo_url,noindex"`
Messages []ModelB `datastore:"-"`
}
// Load is implemented for the PropertyLoaderSaver interface.
func (s *ModelA) Load(c <-chan datastore.Property) (err error) {
f := make(chan datastore.Property, 32)
errc := make(chan error, 1)
defer func() {
if err == nil {
err = <-errc
}
}()
go func() {
defer close(f)
for p := range c {
if p.Name == "bm" {
var b ModelB
err := loadLocalStructuredProperty(&b, []byte(p.Value.(string)))
if err != nil {
errc <- err
return
}
s.Messages = append(s.Messages, b)
} else {
f <- p
}
}
errc <- nil
}()
return datastore.LoadStruct(s, f)
}
我不得不从appengine/datastore
包中复制一堆,因为关键函数没有导出,为了简化我需要复制的代码量,我放弃了对Reference
类型的支持。我在问题跟踪器上打开了一个票证,看看我们是否可以导出loadEntity
函数:https://code.google.com/p/googleappengine/issues/detail?id=10426
import (
"errors"
"time"
"appengine"
"appengine/datastore"
pb "appengine_internal/datastore"
proto "code.google.com/p/goprotobuf/proto"
)
func loadLocalStructuredProperty(dst interface{}, raw_proto []byte) error {
var val pb.EntityProto
err := proto.Unmarshal(raw_proto, &val)
if err != nil {
return err
}
return loadEntity(dst, &val)
}
//Copied from appengine/datastore since its not exported
// loadEntity loads an EntityProto into PropertyLoadSaver or struct pointer.
func loadEntity(dst interface{}, src *pb.EntityProto) (err error) {
c := make(chan datastore.Property, 32)
errc := make(chan error, 1)
defer func() {
if err == nil {
err = <-errc
}
}()
go protoToProperties(c, errc, src)
if e, ok := dst.(datastore.PropertyLoadSaver); ok {
return e.Load(c)
}
return datastore.LoadStruct(dst, c)
}
func protoToProperties(dst chan<- datastore.Property, errc chan<- error, src *pb.EntityProto) {
defer close(dst)
props, rawProps := src.Property, src.RawProperty
for {
var (
x *pb.Property
noIndex bool
)
if len(props) > 0 {
x, props = props[0], props[1:]
} else if len(rawProps) > 0 {
x, rawProps = rawProps[0], rawProps[1:]
noIndex = true
} else {
break
}
var value interface{}
if x.Meaning != nil && *x.Meaning == pb.Property_INDEX_VALUE {
value = indexValue{x.Value}
} else {
var err error
value, err = propValue(x.Value, x.GetMeaning())
if err != nil {
errc <- err
return
}
}
dst <- datastore.Property{
Name: x.GetName(),
Value: value,
NoIndex: noIndex,
Multiple: x.GetMultiple(),
}
}
errc <- nil
}
func fromUnixMicro(t int64) time.Time {
return time.Unix(t/1e6, (t%1e6)*1e3)
}
// propValue returns a Go value that combines the raw PropertyValue with a
// meaning. For example, an Int64Value with GD_WHEN becomes a time.Time.
func propValue(v *pb.PropertyValue, m pb.Property_Meaning) (interface{}, error) {
switch {
case v.Int64Value != nil:
if m == pb.Property_GD_WHEN {
return fromUnixMicro(*v.Int64Value), nil
} else {
return *v.Int64Value, nil
}
case v.BooleanValue != nil:
return *v.BooleanValue, nil
case v.StringValue != nil:
if m == pb.Property_BLOB {
return []byte(*v.StringValue), nil
} else if m == pb.Property_BLOBKEY {
return appengine.BlobKey(*v.StringValue), nil
} else {
return *v.StringValue, nil
}
case v.DoubleValue != nil:
return *v.DoubleValue, nil
case v.Referencevalue != nil:
return nil, errors.New("Not Implemented!")
}
return nil, nil
}
// indexValue is a Property value that is created when entities are loaded from
// an index, such as from a projection query.
//
// Such Property values do not contain all of the metadata required to be
// faithfully represented as a Go value, and are instead represented as an
// opaque indexValue. Load the properties into a concrete struct type (e.g. by
// passing a struct pointer to Iterator.Next) to reconstruct actual Go values
// of type int, string, time.Time, etc.
type indexValue struct {
value *pb.PropertyValue
}
某人1的解决方案效果很好,但我有数百万个实体,不想将它们全部重新放置(将keep_keys=True添加到LocalStructuredProperty)。
因此,我创建了一个EntityProto
的精简版本,它消除了对键和路径等的依赖......只需将pb.EntityProto
替换为LocalEntityProto
,现有的python编写的实体应该加载正常(我正在使用PropertyLoadSaver作为嵌套结构)。
免责声明:我只是用它来从 Go 中读取 - 我还没有尝试写回相同的实体以查看它们是否仍在 Python 中加载。
import pb "google.golang.org/appengine/internal/datastore"
import proto "github.com/golang/protobuf/proto"
type LocalEntityProto struct {
Kind *pb.EntityProto_Kind `protobuf:"varint,4,opt,name=kind,enum=appengine.EntityProto_Kind" json:"kind,omitempty"`
KindUri *string `protobuf:"bytes,5,opt,name=kind_uri" json:"kind_uri,omitempty"`
Property []*pb.Property `protobuf:"bytes,14,rep,name=property" json:"property,omitempty"`
RawProperty []*pb.Property `protobuf:"bytes,15,rep,name=raw_property" json:"raw_property,omitempty"`
Rank *int32 `protobuf:"varint,18,opt,name=rank" json:"rank,omitempty"`
XXX_unrecognized []byte `json:"-"`
}
func (m *LocalEntityProto) Reset() { *m = LocalEntityProto{} }
func (m *LocalEntityProto) String() string { return proto.CompactTextString(m) }
func (*LocalEntityProto) ProtoMessage() {}
func (m *LocalEntityProto) GetKind() pb.EntityProto_Kind {
if m != nil && m.Kind != nil {
return *m.Kind
}
return pb.EntityProto_GD_CONTACT
}
func (m *LocalEntityProto) GetKindUri() string {
if m != nil && m.KindUri != nil {
return *m.KindUri
}
return ""
}
func (m *LocalEntityProto) GetProperty() []*pb.Property {
if m != nil {
return m.Property
}
return nil
}
func (m *LocalEntityProto) GetRawProperty() []*pb.Property {
if m != nil {
return m.RawProperty
}
return nil
}
func (m *LocalEntityProto) GetRank() int32 {
if m != nil && m.Rank != nil {
return *m.Rank
}
return 0
}