在 Crystal 中线程化 HTTP 请求

Question

我有需要"并行"运行的代码(不是真的，我知道Crystal不支持并行(。

require "http/client"
thread_count = 4
resps = [] of HTTP::Client::Response
mutex = Thread::Mutex.new
urls = [] of String
(1..10).each { |i| urls << "http://httpbin.org/delay#{i}"}
threads = Array.new(thread_count) {
  Thread.new do
    while url = mutex.synchronize { urls.pop? }
      resp = HTTP::Client.get(url)
      mutex.synchronize { resps << resp }
    end
  end
}
threads.map(&.join)

Thread类的源文件说不要使用它。无论如何，此代码不适用于HTTP::Client。

Answer 1

使用 spawn ：

require "http/client"
require "json"
WORKER_COUNT = 4
to_fetch = Channel(String?).new(WORKER_COUNT)
responses = Channel(HTTP::Client::Response?).new(WORKER_COUNT)
WORKER_COUNT.times do
  spawn do
    loop do
      url = to_fetch.receive
      responses.send url ? HTTP::Client.get(url) : nil
      break unless url
    end
  end
end
spawn do
  10.times do |i|
    to_fetch.send "http://httpbin.org/delay/#{i}"
  end
  WORKER_COUNT.times do
    to_fetch.send nil
  end
end
start = Time.local
worker_done_count = 0
loop do
  response = responses.receive
  if response
    puts "#{Time.local - start}: fetched #{JSON.parse(response.body)["url"]}"
  else
    worker_done_count += 1
    break if worker_done_count == WORKER_COUNT
  end
end