首先是一点背景;GAE 应用程序由第三方开发，为某些嵌入式硬件提供接口，以通过移动网络将数据保存到云中。数据通过 HTTP 作为 JSON 对象从设备发布。GAE 上的应用程序是用 python 编写的，我可以完全访问源代码(即使我不完全理解它！
数据字段"点"是一个固定长度的项目数组，带有位置、日期戳和两个数字。
因此，单个点的数据模型是(为了安全起见而混淆(：
class MeasurementPoint(EndpointsModel):
Val1 = ndb.FloatProperty()
Val2 = ndb.FloatProperty()
timestamp = ndb.DateTimeProperty(auto_now_add=True)
loc = ndb.GeoPtProperty()
对于封装类(再次混淆(：
class CompletedReading(EndpointsModel):
_message_fields_schema = ('id', 'inspection_id', 'timestamp',
'points',
'Val1_average', 'Val2_average', 'Val1_min',
'location_id',
'organisation_id', )
timestamp = ndb.DateTimeProperty(auto_now_add=True)
points = ndb.LocalStructuredProperty(Measurement, repeated=True, compressed=True)
Val1_average = ndb.FloatProperty()
Val2_average = ndb.FloatProperty()
Val1_min = ndb.FloatProperty()
... hopefully you get the idea.
发布的 JSON 如下所示：
{
"points": [
{
"Val1": 999,
"Val2": 319160.3,
"timestamp": "2020-03-03T15:56:01.000000",
"location": {
"lat": 52.75024,
"lon": -1.919412
}
},
{
"Val1": 999,
"Val2": 319160.3,
"timestamp": "2020-03-03T15:56:02.000000",
"location": {
"lat": 52.75024,
"lon": -1.919412
}
},
..... n sets of "points" ( always a fixed size array )
],
"organisation_id": "5634161670881280",
"location_id": "5638358357245952",
"timestamp": "2020-03-03T15:55:58.000000"
}
所以。。。我现在将这些数据存储在数据库中，我正在编写一个.net应用程序(在VB中，但它很容易是C#(，并且我正在使用Google.Cloud.Datastore.V1API来访问它。到目前为止，这一切都有效，我可以使用以下简单代码列出CompleteReading实体：
Imports Google.Cloud.Datastore.V1
Imports Google.Protobuf
Public Class Form1
Private Sub Button1_Click(sender As Object, e As EventArgs) Handles Button1.Click
Dim ProjectId As String = "xxxxxxxxxx" ' Obfuscated for security
Dim db As DatastoreDb
Dim kind As String = "CopmletedReading"
Dim q As Query
Dim Entry As Entity
' In order for this to work you have to set the environment variable
' GOOGLE_APPLICATION_CREDENTIALS= <Path to zzzzzzzzzzzz.json> (Obfuscated for security)
Environment.SetEnvironmentVariable("GOOGLE_APPLICATION_CREDENTIALS", "zzzzzzzzzzzz.json")
db = DatastoreDb.Create(ProjectId)
q = New Query(kind)
Label1.Text = ""
For Each Entry In db.RunQueryLazily(q)
Label1.Text = Label1.Text & vbCrLf & vbCrLf & vbCrLf & vbCrLf &
"ToString:" & vbCrLf & Entry.ToString & vbCrLf & vbCrLf &
"Key:" & vbCrLf & Entry.Key.ToString & vbCrLf & vbCrLf &
"Properties:" & vbCrLf & Entry.Properties().ToString & vbCrLf & vbCrLf &
"name:" & vbCrLf & Entry.Item("points").ToString
Next
End Sub
End Class
导致每个实体的以下输出....
ToString:
{ "key": { "partitionId": { "projectId": "xxxxxxxxxx" }, "path": [ { "kind": "CompletedReading", "id": "6320665133056000" } ] }, "properties": { "Val1_average": { "doubleValue": 422 }, "Val2_average": { "doubleValue": 278 }, "organisation_ref": { "keyValue": { "partitionId": { "projectId": "xxxxxxxxxx" }, "path": [ { "kind": "Organisation", "id": "5629499534213120" } ] } }, "location_id": { "keyValue": { "partitionId": { "projectId": "xxxxxxxxxx" }, "path": [ { "kind": location", "id": "4922041111150592" } ] } }, "points": { "arrayValue": { "values": [ { "meaning": 22, "blobValue": "eJwrkpXiy8gvyqzKzytJzInPKa1QYNDiVGQAgQ8VDkUqHJxSHDn5yYklmfl5QCkRbcPVjjNu713g5WB5d9oJ1vQHEgd0iqQ42KU4SzJzU4sLEnMLQCZwLNu+ZcuOzxeYiqSleMpSi0oyk9GNX1DoAAD8CChx", "excludeFromIndexes": true }, { "meaning": 22, "blobValue": "eJwrkpXiy8gvyqzKzytJzInPKa1QYNDiVGQAgYIqhyIVDk4pjpz85MSSzPw8oJSItuFqxxm39y7wcrB8HMzQk/5A4oBOkRQHuxRnSWZuanFBYm4ByASOXdu3bNnx+QJTkbQUT1lqUUlmMrrxDwocAK8uJ8g=", "excludeFromIndexes": true }, { "meaning": 22, "blobValue": "eJwrkpXiy8gvyqzKzytJzInPKa1QYNDiVGQAgQPVDkUqHJxSHDn5yYklmfl5QCkRbcM3Bmp/9y7wcrB8KWAhlPFA4oBOkRQHuxRnSWZuanFBYm4ByASOXdu3bNnx+QJTkbQUT1lqUUlmMrrxCwodAKVhJzw=", "excludeFromIndexes": true } ] } }, "timestamp": { "timestampValue": "2016-12-14T10:08:47Z" }, "Val1_min": { "doubleValue": 399 } } }
Key:
{ "partitionId": { "projectId": "xxxxxxxxxx" }, "path": [ { "kind": "CompletedReading", "id": "6320665133056000" } ] }
Properties:
{ "Val1_average": { "doubleValue": 422 }, "Val2_average": { "doubleValue": 278 }, "organisation_ref": { "keyValue": { "partitionId": { "projectId": "xxxxxxxxxx" }, "path": [ { "kind": "Organisation", "id": "5629499534213120" } ] } }, "location_id": { "keyValue": { "partitionId": { "projectId": "xxxxxxxxxx" }, "path": [ { "kind": "location", "id": "4922041111150592" } ] } }, "points": { "arrayValue": { "values": [ { "meaning": 22, "blobValue": "eJwrkpXiy8gvyqzKzytJzInPKa1QYNDiVGQAgQ8VDkUqHJxSHDn5yYklmfl5QCkRbcPVjjNu713g5WB5d9oJ1vQHEgd0iqQ42KU4SzJzU4sLEnMLQCZwLNu+ZcuOzxeYiqSleMpSi0oyk9GNX1DoAAD8CChx", "excludeFromIndexes": true }, { "meaning": 22, "blobValue": "eJwrkpXiy8gvyqzKzytJzInPKa1QYNDiVGQAgYIqhyIVDk4pjpz85MSSzPw8oJSItuFqxxm39y7wcrB8HMzQk/5A4oBOkRQHuxRnSWZuanFBYm4ByASOXdu3bNnx+QJTkbQUT1lqUUlmMrrxDwocAK8uJ8g=", "excludeFromIndexes": true }, { "meaning": 22, "blobValue": "eJwrkpXiy8gvyqzKzytJzInPKa1QYNDiVGQAgQPVDkUqHJxSHDn5yYklmfl5QCkRbcM3Bmp/9y7wcrB8KWAhlPFA4oBOkRQHuxRnSWZuanFBYm4ByASOXdu3bNnx+QJTkbQUT1lqUUlmMrrxCwodAKVhJzw=", "excludeFromIndexes": true } ] } }, "timestamp": { "timestampValue": "2016-12-14T10:08:47Z" }, "Val1_min": { "doubleValue": 399 } }
name:
{ "arrayValue": { "values": [ { "meaning": 22, "blobValue": "eJwrkpXiy8gvyqzKzytJzInPKa1QYNDiVGQAgQ8VDkUqHJxSHDn5yYklmfl5QCkRbcPVjjNu713g5WB5d9oJ1vQHEgd0iqQ42KU4SzJzU4sLEnMLQCZwLNu+ZcuOzxeYiqSleMpSi0oyk9GNX1DoAAD8CChx", "excludeFromIndexes": true }, { "meaning": 22, "blobValue": "eJwrkpXiy8gvyqzKzytJzInPKa1QYNDiVGQAgYIqhyIVDk4pjpz85MSSzPw8oJSItuFqxxm39y7wcrB8HMzQk/5A4oBOkRQHuxRnSWZuanFBYm4ByASOXdu3bNnx+QJTkbQUT1lqUUlmMrrxDwocAK8uJ8g=", "excludeFromIndexes": true }, { "meaning": 22, "blobValue": "eJwrkpXiy8gvyqzKzytJzInPKa1QYNDiVGQAgQPVDkUqHJxSHDn5yYklmfl5QCkRbcM3Bmp/9y7wcrB8KWAhlPFA4oBOkRQHuxRnSWZuanFBYm4ByASOXdu3bNnx+QJTkbQUT1lqUUlmMrrxCwodAKVhJzw=", "excludeFromIndexes": true } ] } }
这就是我卡住的点。我不熟悉 GAE/Datastore/Python 或 Google API。我已经尝试尝试将 blob 反序列化为 VB 类，但没有成功。我在 API 中找不到对LocalStructuredProperty的任何引用，因此我无法直接处理它。
我的"最后一个想法"是尝试在 Web 服务器上编写一些 python 以将数据存储读回合适的对象，然后将数据作为 JSON 在我的 VB 应用程序中触发，但我真的希望能够从 VB 访问数据存储中的对象并取回我的点数组。
谁能建议我从这里去哪里？？谢谢你的关注。
目前这只是部分答案....
实际上，该字段被存储为 ZLIB 压缩字符数组。可能有一种更简单的方法可以实现我所做的事情，但这在一定程度上有效;
将 Google ByteString 转换为 VB 字节数组。 从那里将其转换为MemoryStream并解压缩它。 使用 Unpack(( 方法编写一个类来摆弄生成的未压缩流。
我不得不对相当多的东西进行逆向工程，唯一不太有效的是日期戳字段，(目前(我无法弄清楚它是如何存储的。
这是我是如何做到的。不漂亮，因为它仍在进行中，但这让我在这条路线上走得更远;
一、类
' Class to encapsulate a POINT object stored in the database.
Private Class OnePoint
Dim Val1 As Double
Dim Val2 As Double
Dim timestamp As DateTime
Dim lat As Double
Dim lon As Double
Public Sub Unpack(bs As Stream)
Dim b As Byte
Dim b1 As Byte
Dim i As Integer
Dim bArry As Byte()
Dim d As Double
ReDim bArry(20)
' 3 Currently unknown bytes.
bs.ReadByte()
bs.ReadByte()
bs.ReadByte()
b = bs.ReadByte
Debug.Assert(b = &H4) ' Length of the string "Val1"
For i = 0 To b - 1
b1 = bs.ReadByte()
bArry(i) = b1
Next
'debug.assert (check the string is what we expect)
b = bs.ReadByte()
Debug.Assert(b = &H20) ' Always a space
b = bs.ReadByte()
Debug.Assert(b = &H0) ' Always a NULL
b = bs.ReadByte()
Debug.Assert(b = &H2A) ' Always 0x2A
b = bs.ReadByte()
Debug.Assert(b = &H9) ' Always 0x09
b = bs.ReadByte()
Debug.Assert(b = &H21) ' Always 0x21
' Now read the "Val1" value as an 8 byte number LSB..MSB
For i = 0 To 7
b1 = bs.ReadByte()
bArry(i) = b1
Next
If (BitConverter.IsLittleEndian = False) Then
' This computer byte order is not as per the data we have, so reverse the array.
Array.Reverse(bArry)
End If
hLux = BitConverter.ToDouble(bArry, 0)
'Next bit is the location....
b = bs.ReadByte()
Debug.Assert(b = &H72) ' Always 0x72
b = bs.ReadByte()
Debug.Assert(b = &H24) ' Always 0x24
b = bs.ReadByte()
Debug.Assert(b = &H8) ' Always 0x08
b = bs.ReadByte()
Debug.Assert(b = &H9) ' Always 0x09
b = bs.ReadByte()
Debug.Assert(b = &H1A) ' Always 0x1A
b = bs.ReadByte
Debug.Assert(b = &H8) ' Length of the string "location"
For i = 0 To b - 1
b1 = bs.ReadByte()
bArry(i) = b1
Next
'debug.assert (check the string is what we expect)
b = bs.ReadByte()
Debug.Assert(b = &H20) ' Always a space
b = bs.ReadByte()
Debug.Assert(b = &H0) ' Always a NULL
b = bs.ReadByte()
Debug.Assert(b = &H2A) ' Always 0x2A
b = bs.ReadByte()
Debug.Assert(b = &H14) ' Always 0x14
b = bs.ReadByte()
Debug.Assert(b = &H2B) ' Always 0x2B
b = bs.ReadByte()
Debug.Assert(b = &H31) ' Always 0x31
' Now read the latitude value as an 8 byte number LSB..MSB
For i = 0 To 7
b1 = bs.ReadByte()
bArry(i) = b1
Next
If (BitConverter.IsLittleEndian = False) Then
' This computer byte order is not as per the data we have, so reverse the array.
Array.Reverse(bArry)
End If
lat = BitConverter.ToDouble(bArry, 0)
b = bs.ReadByte()
Debug.Assert(b = &H39) ' Always 0x39
' Now read the longitude value as an 8 byte number LSB..MSB
For i = 0 To 7
b1 = bs.ReadByte()
bArry(i) = b1
Next
If (BitConverter.IsLittleEndian = False) Then
' This computer byte order is not as per the data we have, so reverse the array.
Array.Reverse(bArry)
End If
lon = BitConverter.ToDouble(bArry, 0)
b = bs.ReadByte()
Debug.Assert(b = &H2C) ' Always 0x2C
b = bs.ReadByte()
Debug.Assert(b = &H72) ' Always 0x72
b = bs.ReadByte()
Debug.Assert(b = &H1A) ' Always 0x1A
b = bs.ReadByte()
Debug.Assert(b = &H8) ' Always 0x08
b = bs.ReadByte()
Debug.Assert(b = &H7) ' Always 0x07
b = bs.ReadByte()
Debug.Assert(b = &H1A) ' Always 0x1A
b = bs.ReadByte()
Debug.Assert(b = &H9) ' Length of the string "timestamp"
For i = 0 To b - 1
b1 = bs.ReadByte()
bArry(i) = b1
Next
'debug.assert (check the string is what we expect)
' FOR NOW, SKIP THIS FIELD
For i = 0 To 15
b1 = bs.ReadByte()
Next
b = bs.ReadByte()
Debug.Assert(b = &H4) ' Length of the string "Val2"
For i = 0 To b - 1
b1 = bs.ReadByte()
bArry(i) = b1
Next
'debug.assert (check the string is what we expect)
b = bs.ReadByte()
Debug.Assert(b = &H20) ' Always a space
b = bs.ReadByte()
Debug.Assert(b = &H0) ' Always a NULL
b = bs.ReadByte()
Debug.Assert(b = &H2A) ' Always 0x2A
b = bs.ReadByte()
Debug.Assert(b = &H9) ' Always 0x09
b = bs.ReadByte()
Debug.Assert(b = &H21) ' Always 0x21
' Now read the "Val2"value as an 8 byte number LSB..MSB
For i = 0 To 7
b1 = bs.ReadByte()
bArry(i) = b1
Next
If (BitConverter.IsLittleEndian = False) Then
' This computer byte order is not as per the data we have, so reverse the array.
Array.Reverse(bArry)
End If
vLux = BitConverter.ToDouble(bArry, 0)
' Make sure the number of bytes read matches what we expeted....
Debug.Assert(bs.Position = 126)
End Sub
End Class
然后我打开包装的方式...
Dim o As Object = Entry.Properties("points").ArrayValue.Values ' Google.Protobuf.Collections.RepeatedField(Of Google.Cloud.Datastore.V1.Value)
Dim i As Integer
Dim b As Google.Protobuf.ByteString
Dim x As Byte()
Dim Finalresult As Byte()
' o points to the object which is currently a list of 15 compressed inspection points.
For i = 0 To o.count - 1
b = o(i).BlobValue ' The compressed array as a bytestring
ReDim x(b.Length) ' Make some space in an array of bytes....
b.CopyTo(x, 0) ' Copy from Google.Protobuf.ByteString to VB byte array
' Now x is an array of bytes the size of the BLOB...
' ... but it will be a compressed ZLIB format....
' If this is indeed the case then the first two bytes will be 0x78 0x9C
' ... and by debuginng through we can see that they are !!
'
' Unfortunately "Deflate" requires a stream for input, not a byte array.
' So convert the byte array to a stream so that we can pass that to Deflate....
'
' There may be a mode direct way to convert from a Google.Protobuf.ByteString to a stream,
' but this works.
'
' When dealing with memorystreams you have to use the decorator 'Using' to ensure correct object disposal when we're done...
Using myms As New MemoryStream(x)
Using DecompressedMemoryStream As New MemoryStream()
'#Const EITHER = 1
#If EITHER Then
' Either of these methods works....this one relies on NOT having the header
myms.ReadByte() ' Throw away the header
myms.ReadByte() ' Throw away the header
Using DecompressionStream As New DeflateStream(myms, CompressionMode.Decompress)
#Else
' Either of these methods works....this one relies on having the header
Using DecompressionStream As New GZipStream(myms, CompressionMode.Decompress)
#End If
DecompressionStream.CopyTo(DecompressedMemoryStream)
Finalresult = DecompressedMemoryStream.ToArray()
End Using
End Using
End Using
Debug.Print(Finalresult.Length)
If (FinalResult.Length == 126)
Dim ms As New MemoryStream(Finalresult)
p.Unpack(ms)
End If
Next
..如果我弄清楚任何未知字节是什么，或者我可以解码时间戳，我将发布后续答案，但现在我可以做我需要做的事情。
感谢您的观看！