为什么在编译时生成一个约有30K条目的静态HashMap会消耗这么多资源

Question

我正在尝试编写一个build.rs脚本，该脚本创建一个最新的HashMap，将MAC地址的前6个字符映射到相应的供应商。

它有29231个键值对，这导致cargo check在我的源代码上花费了7分钟以上。在此之前，还不到20秒。它还使用了我笔记本电脑上所有8GB的RAM，在这7-8分钟内我无法使用它。

我认为这要么是rustc/cargo错误，要么我做错了什么，我很确定是后者。生成这样的代码的正确方法是什么？

main.rs

use std::collections::{HashMap, HashSet};
use rustc_hash::{FxHashMap, FxHashSet, FxHasher};
type CustomHasher = BuildHasherDefault<FxHasher>;
include!(concat!(env!("OUT_DIR"), "/map_oui.rs"));

map_oui.rs

#[rustfmt::skip]
lazy_static! {
static ref MAP_MACS: FxHashMap<&'static [u8; 6], &'static str> = {
let mut map_macs = HashMap::with_capacity_and_hasher(29231, CustomHasher::default());
map_macs.insert(b"002272", "American Micro-Fuel Device Corp.");
map_macs.insert(b"00D0EF", "IGT");
//...

build.rs

use std::env;
use std::fs::File;
use std::io::prelude::*;
use std::io::{BufReader, BufWriter};
use std::path::Path;
fn main() {
let out_dir = env::var_os("OUT_DIR").unwrap();
let dest_path = Path::new(&out_dir).join("map_oui.rs");
let handle = File::create(dest_path).unwrap();
let mut writer = BufWriter::new(handle);
let response = ureq::get("http://standards-oui.ieee.org/oui.txt")
.call()
.expect("Conection Error");
let mut reader = BufReader::new(response.into_reader());
let mut line = Vec::new();
writer
.write(
b"#[rustfmt::skip]
lazy_static! {
static ref MAP_MACS: FxHashMap<&'static [u8; 6], &'static str> = {
let mut map_macs = HashMap::with_capacity_and_hasher(29231, CustomHasher::default());n",
)
.unwrap();
loop {
match reader.read_until('n' as u8, &mut line) {
Ok(bytes_read) => {
if bytes_read == 0 {
break;
}
if line.get(12..=18).map_or(false, |s| s == b"base 16") {
let mac_oui = String::from_utf8_lossy(&line[0..6]);
let vendor = String::from_utf8_lossy(&line[22..]);
writer.write(b"    map_macs.insert(b"").unwrap();
writer.write(mac_oui.as_bytes()).unwrap();
writer.write(b"", "").unwrap();
writer.write(vendor.trim().as_bytes()).unwrap();
writer.write(b"");n").unwrap();
}
line.clear();
}
Err(_) => (),
}
}
writer
.write(
b"    map_macs
};
}
",
)
.unwrap();
writer.flush().unwrap();
println!("cargo:rerun-if-changed=build.rs");
}

Answer 1

我遵循了@Thomas和@Shepmaster的建议，结果奏效了。目前build.rs生成一个const MAP_MACS: [([u8; 6], &str); 29246]，我围绕数组的二进制搜索编写了一个名为vendor_lookup的包装函数。不过，最好知道如何将HashMap与自定义Hasher一起使用。

main.rs

include!(concat!(env!("OUT_DIR"), "/map_oui.rs"));
fn vendor_lookup(mac_oui: &[u8; 6]) -> &'static str {
let idx = MAP_MACS
.binary_search_by(|probe| probe.0.cmp(mac_oui))
.unwrap(); // this should be a `?`
MAP_MACS[idx].1
}
fn main() {
assert_eq!(vendor_lookup(b"4C3C16"), "Samsung Electronics Co.,Ltd");
}

map_oui.rs

const MAP_MACS: [([u8; 6], &str); 29246] = [
([48, 48, 48, 48, 48, 48], "XEROX CORPORATION"),
([48, 48, 48, 48, 48, 49], "XEROX CORPORATION"),
([48, 48, 48, 48, 48, 50], "XEROX CORPORATION"),
//---snip---
]

build.rs

use std::env;
use std::fs::File;
use std::io::prelude::*;
use std::io::{BufReader, BufWriter};
use std::path::Path;
fn main() {
let response = ureq::get("http://standards-oui.ieee.org/oui.txt")
.call()
.expect("Conection Error");
let mut reader = BufReader::new(response.into_reader());
let mut data: Vec<(Vec<u8>, String)> = Vec::new();
let mut line = Vec::new();
while reader.read_until(b'n', &mut line).unwrap() != 0 {
if line.get(12..=18).map_or(false, |s| s == b"base 16") {
let mac_oui = line[0..6].to_owned();
let vendor = String::from_utf8_lossy(&line[22..]).trim().to_owned();
data.push((mac_oui, vendor));
}
line.clear();
}
data.sort_unstable();
let out_dir = env::var_os("OUT_DIR").unwrap();
let dest_path = Path::new(&out_dir).join("map_oui.rs");
let handle = File::create(dest_path).unwrap();
let mut writer = BufWriter::new(handle);
writeln!(
&mut writer,
"const MAP_MACS: [([u8; 6], &str); {}] = [",
data.len()
)
.unwrap();
for (key, value) in data {
writeln!(&mut writer, "    ({:?}, "{}"),", key, value).unwrap();
}
writeln!(&mut writer, "];").unwrap();
writer.flush().unwrap();
println!("cargo:rerun-if-changed=build.rs");
}