使用可变长度数据和其他JSON对象的切片对JSON进行解组

Question

我有一个关于Go如何处理解组复杂/嵌套JSON的问题，因为在解组之前，我必须在结构中创建整个数据结构。

我正在使用遵循此格式的JSON文件(它是NVD数据库，如果有帮助，请参考此处https://nvd.nist.gov/vuln/data-feeds):

"CVE_data_type" : "CVE",
"CVE_data_format" : "MITRE",
"CVE_data_version" : "4.0",
"CVE_data_numberOfCVEs" : "1085",
"CVE_data_timestamp" : "2021-02-24T17:00Z",
"CVE_Items" : [ {
"cve" : {
"data_type" : "CVE",
"data_format" : "MITRE",
"data_version" : "4.0",
"CVE_data_meta" : {
"ID" : "CVE-2011-0762",
"ASSIGNER" : "[e-mail]"
},
"problemtype" : {
"problemtype_data" : [ {
"description" : [ {
"lang" : "en",
"value" : "CWE-399"
} ]
} ]
},
"references" : {
"reference_data" : [ {
"url" : "ftp://vsftpd.beasts.org/users/cevans/untar/vsftpd-2.3.4/Changelog",
"name" : "ftp://vsftpd.beasts.org/users/cevans/untar/vsftpd-2.3.4/Changelog",
"refsource" : "CONFIRM",
"tags" : [ ]
}, {
"url" : "http://bugs.debian.org/cgi-bin/bugreport.cgi?bug=622741",
"name" : "http://bugs.debian.org/cgi-bin/bugreport.cgi?bug=622741",
"refsource" : "CONFIRM",
"tags" : [ ]
}, {
"url" : "http://cxib.net/stuff/vspoc232.c",
"name" : "http://cxib.net/stuff/vspoc232.c",
"refsource" : "MISC",
"tags" : [ "Exploit" ]
}, {
"url" : "http://jvn.jp/en/jp/JVN37417423/index.html",
"name" : "JVN#37417423",
"refsource" : "JVN",
"tags" : [ ]
}, {
"url" : "http://lists.fedoraproject.org/pipermail/package-announce/2011-March/055881.html",
"name" : "FEDORA-2011-2615",
"refsource" : "FEDORA",
"tags" : [ ]
}, {
"url" : "http://lists.fedoraproject.org/pipermail/package-announce/2011-March/055882.html",
"name" : "FEDORA-2011-2590",
"refsource" : "FEDORA",
"tags" : [ ]
}, {
"url" : "http://lists.fedoraproject.org/pipermail/package-announce/2011-March/055957.html",
"name" : "FEDORA-2011-2567",
"refsource" : "FEDORA",
"tags" : [ ]
}, {
"url" : "http://lists.opensuse.org/opensuse-security-announce/2011-05/msg00005.html",
"name" : "SUSE-SR:2011:009",
"refsource" : "SUSE",
"tags" : [ ]
}, {
"url" : "http://marc.info/?l=bugtraq&m=133226187115472&w=2",
"name" : "HPSBMU02752",
"refsource" : "HP",
"tags" : [ ]
}, {
"url" : "http://securityreason.com/achievement_securityalert/95",
"name" : "20110301 vsftpd 2.3.2 remote denial-of-service",
"refsource" : "SREASONRES",
"tags" : [ "Exploit" ]
}, {
"url" : "http://securityreason.com/securityalert/8109",
"name" : "8109",
"refsource" : "SREASON",
"tags" : [ ]
}, {
"url" : "http://www.debian.org/security/2011/dsa-2305",
"name" : "DSA-2305",
"refsource" : "DEBIAN",
"tags" : [ ]
}, {
"url" : "http://www.exploit-db.com/exploits/16270",
"name" : "16270",
"refsource" : "EXPLOIT-DB",
"tags" : [ ]
}, {
"url" : "http://www.kb.cert.org/vuls/id/590604",
"name" : "VU#590604",
"refsource" : "CERT-VN",
"tags" : [ "US Government Resource" ]
}, {
"url" : "http://www.mandriva.com/security/advisories?name=MDVSA-2011:049",
"name" : "MDVSA-2011:049",
"refsource" : "MANDRIVA",
"tags" : [ ]
}, {
"url" : "http://www.redhat.com/support/errata/RHSA-2011-0337.html",
"name" : "RHSA-2011:0337",
"refsource" : "REDHAT",
"tags" : [ ]
}, {
"url" : "http://www.securityfocus.com/archive/1/516748/100/0/threaded",
"name" : "20110301 vsftpd 2.3.2 remote denial-of-service",
"refsource" : "BUGTRAQ",
"tags" : [ ]
}, {
"url" : "http://www.securityfocus.com/bid/46617",
"name" : "46617",
"refsource" : "BID",
"tags" : [ "Exploit" ]
}, {
"url" : "http://www.securitytracker.com/id?1025186",
"name" : "1025186",
"refsource" : "SECTRACK",
"tags" : [ ]
}, {
"url" : "http://www.ubuntu.com/usn/USN-1098-1",
"name" : "USN-1098-1",
"refsource" : "UBUNTU",
"tags" : [ ]
}, {
"url" : "http://www.vupen.com/english/advisories/2011/0547",
"name" : "ADV-2011-0547",
"refsource" : "VUPEN",
"tags" : [ ]
}, {
"url" : "http://www.vupen.com/english/advisories/2011/0639",
"name" : "ADV-2011-0639",
"refsource" : "VUPEN",
"tags" : [ ]
}, {
"url" : "http://www.vupen.com/english/advisories/2011/0668",
"name" : "ADV-2011-0668",
"refsource" : "VUPEN",
"tags" : [ ]
}, {
"url" : "http://www.vupen.com/english/advisories/2011/0713",
"name" : "ADV-2011-0713",
"refsource" : "VUPEN",
"tags" : [ ]
}, {
"url" : "https://exchange.xforce.ibmcloud.com/vulnerabilities/65873",
"name" : "vsftpd-vsffilenamepassesfilter-dos(65873)",
"refsource" : "XF",
"tags" : [ ]
} ]
},
"description" : {
"description_data" : [ {
"lang" : "en",
"value" : "The vsf_filename_passes_filter function in ls.c in vsftpd before 2.3.3 allows remote authenticated users to cause a denial of service (CPU consumption and process slot exhaustion) via crafted glob expressions in STAT commands in multiple FTP sessions, a different vulnerability than CVE-2010-2632."
} ]
}
},
"configurations" : {
"CVE_data_version" : "4.0",
"nodes" : [ {
"operator" : "OR",
"cpe_match" : [ {
"vulnerable" : true,
"cpe23Uri" : "cpe:2.3:a:beasts:vsftpd:0.0.1:*:*:*:*:*:*:*"
}, {
"vulnerable" : true,
"cpe23Uri" : "cpe:2.3:a:beasts:vsftpd:0.0.2:*:*:*:*:*:*:*"
}, {
"vulnerable" : true,
"cpe23Uri" : "cpe:2.3:a:beasts:vsftpd:0.0.3:*:*:*:*:*:*:*"
}, {
"vulnerable" : true,
"cpe23Uri" : "cpe:2.3:a:beasts:vsftpd:0.0.4:*:*:*:*:*:*:*"
}, {
"vulnerable" : true,
"cpe23Uri" : "cpe:2.3:a:beasts:vsftpd:0.0.5:*:*:*:*:*:*:*"
}, {
"vulnerable" : true,
"cpe23Uri" : "cpe:2.3:a:beasts:vsftpd:0.0.6:*:*:*:*:*:*:*"
}, {
"vulnerable" : true,
"cpe23Uri" : "cpe:2.3:a:beasts:vsftpd:0.0.7:*:*:*:*:*:*:*"
}, {
"vulnerable" : true,
"cpe23Uri" : "cpe:2.3:a:beasts:vsftpd:0.0.8:*:*:*:*:*:*:*"
}, {
"vulnerable" : true,
"cpe23Uri" : "cpe:2.3:a:beasts:vsftpd:0.0.9:*:*:*:*:*:*:*"
}, {
"vulnerable" : true,
"cpe23Uri" : "cpe:2.3:a:beasts:vsftpd:0.0.10:*:*:*:*:*:*:*"
}, {
"vulnerable" : true,
"cpe23Uri" : "cpe:2.3:a:beasts:vsftpd:0.0.11:*:*:*:*:*:*:*"
}, {
"vulnerable" : true,
"cpe23Uri" : "cpe:2.3:a:beasts:vsftpd:0.0.12:*:*:*:*:*:*:*"
}, {
"vulnerable" : true,
"cpe23Uri" : "cpe:2.3:a:beasts:vsftpd:0.0.13:*:*:*:*:*:*:*"
}, {
"vulnerable" : true,
"cpe23Uri" : "cpe:2.3:a:beasts:vsftpd:0.0.14:*:*:*:*:*:*:*"
}, {
"vulnerable" : true,
"cpe23Uri" : "cpe:2.3:a:beasts:vsftpd:0.0.15:*:*:*:*:*:*:*"
}, {
"vulnerable" : true,
"cpe23Uri" : "cpe:2.3:a:beasts:vsftpd:0.9.0:*:*:*:*:*:*:*"
}, {
"vulnerable" : true,
"cpe23Uri" : "cpe:2.3:a:beasts:vsftpd:0.9.1:*:*:*:*:*:*:*"
}, {
"vulnerable" : true,
"cpe23Uri" : "cpe:2.3:a:beasts:vsftpd:0.9.2:*:*:*:*:*:*:*"
}, {
"vulnerable" : true,
"cpe23Uri" : "cpe:2.3:a:beasts:vsftpd:0.9.3:*:*:*:*:*:*:*"
}, {
"vulnerable" : true,
"cpe23Uri" : "cpe:2.3:a:beasts:vsftpd:1.0.0:*:*:*:*:*:*:*"
}, {
"vulnerable" : true,
"cpe23Uri" : "cpe:2.3:a:beasts:vsftpd:1.0.1:*:*:*:*:*:*:*"
}, {
"vulnerable" : true,
"cpe23Uri" : "cpe:2.3:a:beasts:vsftpd:1.1.0:*:*:*:*:*:*:*"
}, {
"vulnerable" : true,
"cpe23Uri" : "cpe:2.3:a:beasts:vsftpd:1.1.1:*:*:*:*:*:*:*"
}, {
"vulnerable" : true,
"cpe23Uri" : "cpe:2.3:a:beasts:vsftpd:1.1.2:*:*:*:*:*:*:*"
}, {
"vulnerable" : true,
"cpe23Uri" : "cpe:2.3:a:beasts:vsftpd:1.1.3:*:*:*:*:*:*:*"
}, {
"vulnerable" : true,
"cpe23Uri" : "cpe:2.3:a:beasts:vsftpd:1.2.0:*:*:*:*:*:*:*"
}, {
"vulnerable" : true,
"cpe23Uri" : "cpe:2.3:a:beasts:vsftpd:1.2.1:*:*:*:*:*:*:*"
}, {
"vulnerable" : true,
"cpe23Uri" : "cpe:2.3:a:beasts:vsftpd:1.2.2:*:*:*:*:*:*:*"
}, {
"vulnerable" : true,
"cpe23Uri" : "cpe:2.3:a:beasts:vsftpd:2.0.0:*:*:*:*:*:*:*"
}, {
"vulnerable" : true,
"cpe23Uri" : "cpe:2.3:a:beasts:vsftpd:2.0.1:*:*:*:*:*:*:*"
}, {
"vulnerable" : true,
"cpe23Uri" : "cpe:2.3:a:beasts:vsftpd:2.0.2:*:*:*:*:*:*:*"
}, {
"vulnerable" : true,
"cpe23Uri" : "cpe:2.3:a:beasts:vsftpd:2.0.3:*:*:*:*:*:*:*"
}, {
"vulnerable" : true,
"cpe23Uri" : "cpe:2.3:a:beasts:vsftpd:2.0.4:*:*:*:*:*:*:*"
}, {
"vulnerable" : true,
"cpe23Uri" : "cpe:2.3:a:beasts:vsftpd:2.0.5:*:*:*:*:*:*:*"
}, {
"vulnerable" : true,
"cpe23Uri" : "cpe:2.3:a:beasts:vsftpd:2.0.6:*:*:*:*:*:*:*"
}, {
"vulnerable" : true,
"cpe23Uri" : "cpe:2.3:a:beasts:vsftpd:2.0.7:*:*:*:*:*:*:*"
}, {
"vulnerable" : true,
"cpe23Uri" : "cpe:2.3:a:beasts:vsftpd:2.1.0:*:*:*:*:*:*:*"
}, {
"vulnerable" : true,
"cpe23Uri" : "cpe:2.3:a:beasts:vsftpd:2.1.1:*:*:*:*:*:*:*"
}, {
"vulnerable" : true,
"cpe23Uri" : "cpe:2.3:a:beasts:vsftpd:2.1.2:*:*:*:*:*:*:*"
}, {
"vulnerable" : true,
"cpe23Uri" : "cpe:2.3:a:beasts:vsftpd:2.2.0:*:*:*:*:*:*:*"
}, {
"vulnerable" : true,
"cpe23Uri" : "cpe:2.3:a:beasts:vsftpd:2.2.1:*:*:*:*:*:*:*"
}, {
"vulnerable" : true,
"cpe23Uri" : "cpe:2.3:a:beasts:vsftpd:2.2.2:*:*:*:*:*:*:*"
}, {
"vulnerable" : true,
"cpe23Uri" : "cpe:2.3:a:beasts:vsftpd:2.3.0:*:*:*:*:*:*:*"
}, {
"vulnerable" : true,
"cpe23Uri" : "cpe:2.3:a:beasts:vsftpd:2.3.1:*:*:*:*:*:*:*"
}, {
"vulnerable" : true,
"cpe23Uri" : "cpe:2.3:a:beasts:vsftpd:*:*:*:*:*:*:*:*",
"versionEndIncluding" : "2.3.2"
} ]
} ]
},
"impact" : {
"baseMetricV2" : {
"cvssV2" : {
"version" : "2.0",
"vectorString" : "AV:N/AC:L/Au:S/C:N/I:N/A:P",
"accessVector" : "NETWORK",
"accessComplexity" : "LOW",
"authentication" : "SINGLE",
"confidentialityImpact" : "NONE",
"integrityImpact" : "NONE",
"availabilityImpact" : "PARTIAL",
"baseScore" : 4.0
},
"severity" : "MEDIUM",
"exploitabilityScore" : 8.0,
"impactScore" : 2.9,
"obtainAllPrivilege" : false,
"obtainUserPrivilege" : false,
"obtainOtherPrivilege" : false,
"userInteractionRequired" : false
}
},
"publishedDate" : "2011-03-02T20:00Z",
"lastModifiedDate" : "2021-02-19T05:15Z"
}, {
"cve" : {
"data_type" : "CVE",
"data_format" : "MITRE",
"data_version" : "4.0",
"CVE_data_meta" : {
"ID" : "CVE-2011-4362",
"ASSIGNER" : "[e-mail]"
},
"problemtype" : {
"problemtype_data" : [ {
"description" : [ {
"lang" : "en",
"value" : "CWE-189"
} ]
} ]
},
"references" : {
"reference_data" : [ {
"url" : "http://archives.neohapsis.com/archives/bugtraq/2011-12/0167.html",
"name" : "20111224 Lighttpd Proof of Concept code for CVE-2011-4362",
"refsource" : "BUGTRAQ",
"tags" : [ "Broken Link" ]
}, {
"url" : "http://blog.pi3.com.pl/?p=277",
"name" : "http://blog.pi3.com.pl/?p=277",
"refsource" : "MISC",
"tags" : [ "Third Party Advisory" ]
}, {
"url" : "http://download.lighttpd.net/lighttpd/security/lighttpd_sa_2011_01.txt",
"name" : "http://download.lighttpd.net/lighttpd/security/lighttpd_sa_2011_01.txt",
"refsource" : "CONFIRM",
"tags" : [ "Vendor Advisory" ]
}, {
"url" : "http://jvn.jp/en/jp/JVN37417423/index.html",
"name" : "JVN#37417423",
"refsource" : "JVN",
"tags" : [ ]
}, {
"url" : "http://redmine.lighttpd.net/issues/2370",
"name" : "http://redmine.lighttpd.net/issues/2370",
"refsource" : "CONFIRM",
"tags" : [ "Vendor Advisory" ]
}, {
"url" : "http://secunia.com/advisories/47260",
"name" : "47260",
"refsource" : "SECUNIA",
"tags" : [ "Third Party Advisory" ]
}, {
"url" : "http://www.debian.org/security/2011/dsa-2368",
"name" : "DSA-2368",
"refsource" : "DEBIAN",
"tags" : [ "Third Party Advisory" ]
}, {
"url" : "http://www.exploit-db.com/exploits/18295",
"name" : "18295",
"refsource" : "EXPLOIT-DB",
"tags" : [ "Third Party Advisory", "VDB Entry" ]
}, {
"url" : "http://www.openwall.com/lists/oss-security/2011/11/29/13",
"name" : "[oss-security] 20111129 Re: CVE Request: lighttpd/mod_auth out-of-bounds read due to signedness error",
"refsource" : "MLIST",
"tags" : [ "Mailing List", "Third Party Advisory" ]
}, {
"url" : "http://www.openwall.com/lists/oss-security/2011/11/29/8",
"name" : "[oss-security] 20111129 CVE Request: lighttpd/mod_auth out-of-bounds read due to signedness error",
"refsource" : "MLIST",
"tags" : [ "Mailing List", "Third Party Advisory" ]
}, {
"url" : "http://www.securitytracker.com/id?1026359",
"name" : "1026359",
"refsource" : "SECTRACK",
"tags" : [ "Third Party Advisory", "VDB Entry" ]
}, {
"url" : "https://bugzilla.redhat.com/show_bug.cgi?id=758624",
"name" : "https://bugzilla.redhat.com/show_bug.cgi?id=758624",
"refsource" : "CONFIRM",
"tags" : [ "Issue Tracking", "Third Party Advisory" ]
}, {
"url" : "https://exchange.xforce.ibmcloud.com/vulnerabilities/71536",
"name" : "lighttpd-base64-dos(71536)",
"refsource" : "XF",
"tags" : [ "Third Party Advisory", "VDB Entry" ]
} ]
},
"description" : {
"description_data" : [ {
"lang" : "en",
"value" : "Integer signedness error in the base64_decode function in the HTTP authentication functionality (http_auth.c) in lighttpd 1.4 before 1.4.30 and 1.5 before SVN revision 2806 allows remote attackers to cause a denial of service (segmentation fault) via crafted base64 input that triggers an out-of-bounds read with a negative index."
} ]
}
},
"configurations" : {
"CVE_data_version" : "4.0",
"nodes" : [ {
"operator" : "OR",
"cpe_match" : [ {
"vulnerable" : true,
"cpe23Uri" : "cpe:2.3:a:lighttpd:lighttpd:*:*:*:*:*:*:*:*",
"versionStartIncluding" : "1.4.1",
"versionEndExcluding" : "1.4.30"
}, {
"vulnerable" : true,
"cpe23Uri" : "cpe:2.3:a:lighttpd:lighttpd:1.5.0:*:*:*:*:*:*:*"
} ]
}, {
"operator" : "OR",
"cpe_match" : [ {
"vulnerable" : true,
"cpe23Uri" : "cpe:2.3:o:debian:debian_linux:5.0:*:*:*:*:*:*:*"
}, {
"vulnerable" : true,
"cpe23Uri" : "cpe:2.3:o:debian:debian_linux:6.0:*:*:*:*:*:*:*"
}, {
"vulnerable" : true,
"cpe23Uri" : "cpe:2.3:o:debian:debian_linux:7.0:*:*:*:*:*:*:*"
} ]
} ]
},
"impact" : {
"baseMetricV2" : {
"cvssV2" : {
"version" : "2.0",
"vectorString" : "AV:N/AC:L/Au:N/C:N/I:N/A:P",
"accessVector" : "NETWORK",
"accessComplexity" : "LOW",
"authentication" : "NONE",
"confidentialityImpact" : "NONE",
"integrityImpact" : "NONE",
"availabilityImpact" : "PARTIAL",
"baseScore" : 5.0
},
"severity" : "MEDIUM",
"exploitabilityScore" : 10.0,
"impactScore" : 2.9,
"obtainAllPrivilege" : false,
"obtainUserPrivilege" : false,
"obtainOtherPrivilege" : false,
"userInteractionRequired" : false
}
},
"publishedDate" : "2011-12-24T19:55Z",
"lastModifiedDate" : "2021-02-19T05:15Z"
},

正如你所看到的，这里有很多数据，我正在努力把这些数据整理成我可以使用的东西。前几个字段很简单，所以我开始创建这样一个结构体:

type NvdData struct {
CveDataType string `json:"CVE_data_type"`
CveDataFormat string `json:"CVE_data_format"`
CveDataVersion string `json:"CVE_data_version"`
CveDataNumberCves string `json:"CVE_data_numberOfCVEs"`

}

我感到困惑的是，当我们到达"cve_item "字段，该字段包含"CVE"的列表/切片。对象都有自己不同的字段，其中一些是列表，一些不是。

我的问题是，我如何构建我的NvdData结构来处理这个?我是否需要创建一个额外的结构体CVE，其中包含与每个CVE条目相关的所有不同字段，并将该数据结构作为NvdData定义的一部分?比如:

type NvdData struct {
CveDataType string `json:"CVE_data_type"`
CveDataFormat string `json:"CVE_data_format"`
CveDataVersion string `json:"CVE_data_version"`
CveDataNumberCves string `json:"CVE_data_numberOfCVEs"`
CveItems []Cve{} `json:"CVE_Items`
}
type Cve struct {
DataType string `json:"data_type"`
DataFormat string `json:"data_format"`
...
}

明白了吗?然后对于任何后续的数据结构是列表/切片，我将不得不描述为一个结构体以及type为json字段将是该类型的切片?

我还有一个一次性的问题，如果有我不关心的字段，我可以从类型定义中删除它们，它们将被解组过程忽略，或者我也需要描述它们，当我遍历数据时忽略它们?

Answer 1

我看到有一个模式在线:https://csrc.nist.gov/schema/nvd/feed/1.1/nvd_cve_feed_json_1.1.schema

具有this，它定义了一个CVE项(它引用了同一文件中的其他定义)。理论上，您可以使用它来创建各种结构。这里尝试自动生成结构:https://adrianhesketh.com/2016/07/19/json-schema-to-go-struct-generator-roundup/

"def_cve_item": {
"description": "Defines a vulnerability in the NVD data feed.",
"properties": {
"cve": {"$ref": "CVE_JSON_4.0_min_1.1.schema"},
"configurations": {"$ref": "#/definitions/def_configurations"},
"impact": {"$ref": "#/definitions/def_impact"},
"publishedDate": {"type": "string"},
"lastModifiedDate": {"type": "string"}
},
"required": ["cve"]
}

我认为最简单的(虽然不是最健壮的)是将其分解为map[string]interface{}

type CVEItem struct {
CVE              map[string]interface{} `json:"cve"`
Configurations   map[string]interface{} `json:"configurations"`
Impact           map[string]interface{} `json:"impact"`
PublishedDate    string                 `json:"publishedDate"`
LastModifiedDate string                 `json:"lastModifiedDate"`
}
type DataStruct struct {
DataType string    `json:"CVE_data_type"`
CVEItems []CVEItem `json:"CVE_items"`
}
var result DataStruct
err = json.Unmarshal([]byte(byteValue), &result)
if err != nil {
fmt.Println(err)
}

然后，您可以循环遍历各种项并从那里构建具体的结构(通过查看键)。如果有许多可选的或嵌套的项目……那么，是的，这将是一件痛苦的事。