我需要通过 awk 获取日志文件在 1、2、3、...、N分钟内的请求计数。
例如，用户设置一个对应于一分钟间隔($interval=1(的变量 - 输出应该像
09:01 - 09:02
count of requests
09:02 - 09:03
count of requests
...
09:16 - 09:17
count of requests
间隔 2 分钟：
09:01 - 09:03
count of requests
09:03 - 09:05
count of requests
...
09:15 - 09:17
count of requests
我尝试通过 bash 脚本解决分配的任务，但它只工作一分钟：
#!/bin/bash
for ((i=1; i<=17 ;i++))
do
echo "in $i interval"
if [[ "$i" -lt 10 ]] ;then
awk "/2015:09:0$i.*/" access_log | awk 'END{print NR}'
else
awk "/2015:09:$i.*/" access_log | awk 'END{print NR}'
fi
done
输入行示例;
10.1.2.194 (207.46.13.7, 54.239.137.128) - - [08/Oct/2015:09:01:42 +0000] "GET /merlin-image-server/view/756af03a-6dc4-4568-9081-0b6f48d2f9d5/120 HTTP/1.1" 200 2944 0 100220 "-" "Amazon CloudFront" "ajp://10.1.3.202:8009"
10.1.2.194 (78.192.164.23) - - [08/Oct/2015:09:01:42 +0000] "GET /merlin-web-za/cb3c50fb8011691d674f6df81d57e2a7e/web/img/search/serp-bg.png HTTP/1.1" 200 6986 0 1772 "http://www.autotrader.co.za/merlin-web-za/bundles/css/N943289621/bundle.css" "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/45.0.2454.101 Safari/537.36" "ajp://10.1.4.17:8009"
10.1.2.194 (78.192.164.23) - - [08/Oct/2015:09:01:42 +0000] "GET /merlin-web-za/cbad2541a0cf556c96fd6dd70a692636c3/web/images/search/counter.gif HTTP/1.1" 304 - 0 1517 "http://www.autotrader.co.za/makemodel/make/mercedes-benz/model/s-class/search" "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/45.0.2454.101 Safari/537.36" "ajp://10.1.4.17:8009"
10.1.2.194 (78.192.164.23) - - [08/Oct/2015:09:01:42 +0000] "GET /merlin-web-za/cb9ddf6600496d51326b3b02718fd8d1da/web/img/global/new.png HTTP/1.1" 304 - 0 1301 "http://www.autotrader.co.za/makemodel/make/mercedes-benz/model/s-class/search" "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/45.0.2454.101 Safari/537.36" "ajp://10.1.4.17:8009"
10.1.2.194 (78.192.164.23) - - [08/Oct/2015:09:01:42 +0000] "GET /merlin-web-za/web/images/refinements/loader.gif HTTP/1.1" 200 4178 0 1652 "http://www.autotrader.co.za/makemodel/make/mercedes-benz/model/s-class/search" "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/45.0.2454.101 Safari/537.36" "ajp://10.1.4.17:8009"
10.1.2.194 (196.11.233.81) - - [08/Oct/2015:09:01:42 +0000] "GET /merlin-web-za/web/images/search/watchlist-error-icon.png HTTP/1.1" 200 429 0 1631 "http://www.autotrader.co.za/seoregion/kwazulu-natal/makemodel/make/volkswagen/model/polo/bodytype/hatchback/search?sort=PriceAsc&county=KwaZulu-Natal&longitude=31.0292&locationName=Durban&latitude=-29.8579&pageNumber=22" "Mozilla/5.0 (Windows NT 6.1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/45.0.2454.101 Safari/537.36" "ajp://10.1.3.86:8009"
10.1.2.194 (196.15.160.44) - - [08/Oct/2015:09:06:26 +0000] "GET /merlin-web-za/bundles/js/935688859/bundle.js HTTP/1.1" 304 - 0 1476 "http://www.autotrader.co.za/carandcommercialpricerangeszar/85-000-to-99-999/transmissiontype/automatic/carandcommercialpricerangeszar/100-000-to-124-999/carandcommercialpricerangeszar/125-000-to-149-999/search?pageNumber=29" "Mozilla/5.0 (Windows NT 6.1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/47.0.2526.5 Safari/537.36" "ajp://10.1.3.88:8009"
10.1.2.194 (196.15.160.44) - - [08/Oct/2015:09:06:26 +0000] "GET /merlin-web-za/bundles/css/N943289621/bundle.css HTTP/1.1" 304 - 0 1297 "http://www.autotrader.co.za/carandcommercialpricerangeszar/85-000-to-99-999/transmissiontype/automatic/carandcommercialpricerangeszar/100-000-to-124-999/carandcommercialpricerangeszar/125-000-to-149-999/search?pageNumber=29" "Mozilla/5.0 (Windows NT 6.1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/47.0.2526.5 Safari/537.36" "ajp://10.1.3.88:8009"
10.1.2.194 (54.77.132.130) - - [08/Oct/2015:09:06:26 +0000] "GET /merlin-service-search/rest/featuresbyatmmv?atmmvcode=7799 HTTP/1.1" 200 6249 0 10193 "-" "Jakarta Commons-HttpClient/3.1" "ajp://10.1.4.67:8009"
10.1.2.194 (66.249.91.222) - - [08/Oct/2015:09:06:26 +0000] "GET /makemodel/make/OPEL/model/ASTRA/bodytype/Hatchback/search?pageNumber=4 HTTP/1.1" 301 20 0 1860 "-" "Mediapartners-Google" "ajp://10.1.3.203:8009"
10.1.2.194 (197.77.174.86) - - [08/Oct/2015:09:06:26 +0000] "GET /merlin-service-search/rest/dblastupdatetime HTTP/1.1" 200 25 0 2083 "-" "-" "ajp://10.1.3.204:8009"
10.1.2.194 (66.85.139.244, 216.137.44.16) - - [08/Oct/2015:09:06:26 +0000] "GET /merlin-image-server/view/118e4e83-a621-44f1-8c85-8a2e95145055/800 HTTP/1.1" 200 72634 0 89903 "-" "Amazon CloudFront" "ajp://10.1.3.202:8009"
10.1.2.194 (54.246.139.111) - - [08/Oct/2015:09:06:26 +0000] "GET /merlin-service-search/rest/featuresbyatmmv?atmmvcode=27749 HTTP/1.1" 200 70 0 7872 "-" "Jakarta Commons-HttpClient/3.1" "ajp://10.1.3.86:8009"
10.1.2.194 (54.246.139.111) - - [08/Oct/2015:09:06:26 +0000] "GET /merlin-service-search/rest/featuresbyatmmv?atmmvcode=27753 HTTP/1.1" 200 4267 0 11619 "-" "Jakarta Commons-HttpClient/3.1" "ajp://10.1.3.205:8009"
10.1.2.194 (54.77.132.130) - - [08/Oct/2015:09:11:48 +0000] "GET /merlin-service-search/rest/featuresbyatmmv?atmmvcode=11867 HTTP/1.1" 200 348 0 10677 "-" "Jakarta Commons-HttpClient/3.1" "ajp://10.1.3.202:8009"
10.1.2.194 (54.77.132.130) - - [08/Oct/2015:09:11:48 +0000] "GET /merlin-service-search/rest/featuresbyatmmv?atmmvcode=17267 HTTP/1.1" 200 3266 0 7127 "-" "Jakarta Commons-HttpClient/3.1" "ajp://10.1.4.17:8009"
10.1.2.194 (41.208.251.58, 54.240.147.15) - - [08/Oct/2015:09:11:48 +0000] "GET /merlin-image-server/view/916e54c2-2b4a-40b2-8f41-4a0dc8ec0b0e/220 HTTP/1.1" 200 8555 0 111773 "-" "Amazon CloudFront" "ajp://10.1.4.68:8009"
10.1.2.194 (197.77.1.243, 54.240.147.77) - - [08/Oct/2015:09:11:48 +0000] "GET /merlin-image-server/view/8cfd5e47-7d5d-45f6-8380-3276e2c4d552/650 HTTP/1.1" 200 70044 0 164335 "-" "Amazon CloudFront" "ajp://10.1.3.202:8009"
10.1.2.194 (196.26.128.178) - - [08/Oct/2015:09:11:48 +0000] "GET /merlin-web-za/web/images/fav.ico HTTP/1.1" 200 643 0 881 "-" "Mozilla/5.0 (Windows NT 6.3; WOW64; Trident/7.0; rv:11.0) like Gecko" "ajp://10.1.4.67:8009"
10.1.2.194 (196.22.229.4) - - [08/Oct/2015:09:11:48 +0000] "GET /merlin-web-za/bundles/css/N943289621/bundle.css HTTP/1.1" 200 39605 0 13377 "http://www.autotrader.co.za/makemodel/make/AUDI/model/A1/neworused/new/search" "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/45.0.2454.85 Safari/537.36" "ajp://10.1.3.88:8009"
10.1.2.194 (105.210.54.155) - - [08/Oct/2015:09:11:48 +0000] "GET /merlin-web-za/bundles/css/N943289621/bundle.css HTTP/1.1" 304 - 0 918 "http://www.autotrader.co.za/neworused/used/fueltype/petrol/makemodel/make/toyota/model/yaris/makemodel/make/nissan/model/hardbody/model/np300-hardbody/search?sort=PriceAsc" "Mozilla/5.0 (Windows NT 6.1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/45.0.2454.101 Safari/537.36" "ajp://10.1.4.70:8009"
10.1.2.194 (197.79.11.175) - - [08/Oct/2015:09:11:48 +0000] "GET /used-cars/land-rover/freelander-2/2007-land-rover-freelander-2-s-i6-gezina-fpa-8a81839749ae76d9014a4e672e347fe2 HTTP/1.1" 200 30154 0 101659 "http://www.autotrader.co.za/makemodel/make/LAND%20ROVER/model/FREELANDER%202/search?&pageNumber=9" "Mozilla/5.0 (Linux; U; Android 4.1.2; en-gb; GT-N7100 Build/JZO54K) AppleWebKit/534.30 (KHTML, like Gecko) Version/4.0 Mobile Safari/534.30" "ajp://10.1.4.66:8009"
10.1.2.194 (41.189.79.130) - - [08/Oct/2015:09:11:48 +0000] "GET /merlin-web-za/web/images/fav.ico HTTP/1.1" 200 894 0 1400 "http://www.autotrader.co.za/makemodel/make/jeep/model/wrangler/caryearrangeszar/2015/search?keywords=unlimited&sort=PriceAsc&gquery=null" "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/45.0.2454.101 Safari/537.36" "ajp://10.1.3.205:8009"
10.1.2.194 (41.13.72.175) - - [08/Oct/2015:09:11:48 +0000] "GET /merlin-web-za/cb161f2003381435a3347b2603bfd9fa5d/mobile/images/gr.gif HTTP/1.1" 200 413 0 1310 "http://www.autotrader.co.za/makemodel/make/OPEL/model/COMBO/carandcommercialpricerangeszar/under-10000/carandcommercialpricerangeszar/10000-to-24999/carandcommercialpricerangeszar/25000-to-39999/carandcommercialpricerangeszar/40000-to-54999/carandcommercialpricerangeszar/55000-to-69999/carandcommercialpricerangeszar/70000-to-84999/carandcommercialpricerangeszar/85000-to-99999/carandcommercialpricerangeszar/100000-to-124999/carandcommercialpricerangeszar/125000-to-149999/carandcommercialpricerangeszar/150000-to-174999/carandcommercialpricerangeszar/175000-to-199999/search" "Mozilla/5.0 (Linux; Android 4.4.4; SM-A500F Build/KTU84P) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/36.0.1985.534 Mobile Safari/537.36" "ajp://10.1.3.205:8009"
10.1.2.194 (196.210.212.56) - - [08/Oct/2015:09:11:48 +0000] "GET /merlin-web-za/cbad2541a0cf556c96fd6dd70a692636c3/web/images/search/counter.gif HTTP/1.1" 304 - 0 576 "http://www.autotrader.co.za/makemodel/make/SUZUKI/model/SWIFT/search?pageNumber=7" "Mozilla/5.0 (Windows NT 6.1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/45.0.2454.101 Safari/537.36" "ajp://10.1.4.15:8009"
10.1.2.194 (41.13.72.175) - - [08/Oct/2015:09:11:48 +0000] "GET /merlin-web-za/cb53aec1b9de3d5953ed77f3979a4cb960/mobile/images/green.gif HTTP/1.1" 200 1009 0 1339 "http://www.autotrader.co.za/makemodel/make/OPEL/model/COMBO/carandcommercialpricerangeszar/under-10000/carandcommercialpricerangeszar/10000-to-24999/carandcommercialpricerangeszar/25000-to-39999/carandcommercialpricerangeszar/40000-to-54999/carandcommercialpricerangeszar/55000-to-69999/carandcommercialpricerangeszar/70000-to-84999/carandcommercialpricerangeszar/85000-to-99999/carandcommercialpricerangeszar/100000-to-124999/carandcommercialpricerangeszar/125000-to-149999/carandcommercialpricerangeszar/150000-to-174999/carandcommercialpricerangeszar/175000-to-199999/search" "Mozilla/5.0 (Linux; Android 4.4.4; SM-A500F Build/KTU84P) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/36.0.1985.534 Mobile Safari/537.36" "ajp://10.1.3.205:8009"
10.1.2.194 (41.160.30.18, 54.240.147.15) - - [08/Oct/2015:09:11:48 +0000] "GET /merlin-image-server/view/1f5d62d6-a303-4a86-9500-a726e260d845/120 HTTP/1.1" 200 2275 0 53108 "-" "Amazon CloudFront" "ajp://10.1.4.68:8009"
10.1.2.194 (207.46.13.7) - - [08/Oct/2015:09:11:48 +0000] "GET /used-commercial-vehicles/isuzu/npr/1980-isuzu-spr-422-5-ton-port-elizabeth-cfpa-8aa3054c44b1c9500144df47c82e51a2/seoregion/eastern-cape/makemodel/make/isuzu HTTP/1.1" 301 20 0 19664 "-" "Mozilla/5.0 (compatible; bingbot/2.0; +http://www.bing.com/bingbot.htm)" "ajp://10.1.3.204:8009"
10.1.2.194 (54.77.132.130) - - [08/Oct/2015:09:14:43 +0000] "GET /merlin-service-search/rest/featuresbyatmmv?atmmvcode=15395 HTTP/1.1" 200 1296 0 9272 "-" "Jakarta Commons-HttpClient/3.1" "ajp://10.1.3.201:8009"
10.1.2.194 (41.222.51.122, 54.239.183.6) - - [08/Oct/2015:09:14:43 +0000] "GET /merlin-image-server/view/3070c6d7-00b6-4186-b884-a19133ea3a97/200 HTTP/1.1" 200 6267 0 58404 "-" "Amazon CloudFront" "ajp://10.1.4.70:8009"
10.1.2.194 (196.4.0.2, 54.240.157.43) - - [08/Oct/2015:09:14:43 +0000] "GET /merlin-image-server/view/997e28de-6ea4-401d-8960-f28d55675c43/60 HTTP/1.1" 200 1312 0 93228 "-" "Amazon CloudFront" "ajp://10.1.4.66:8009"
10.1.2.194 (41.222.51.122, 54.239.183.52) - - [08/Oct/2015:09:14:43 +0000] "GET /merlin-image-server/view/0685c8e8-2e66-4f27-8ff3-4e45ebaccf6f/200 HTTP/1.1" 200 6359 0 74447 "-" "Amazon CloudFront" "ajp://10.1.3.88:8009"
在此处查看整个文件。
你的要求在几个方面不清楚(请参阅我在你的问题下的评论(，但这是使用 GNU awk 进行时间函数的一般方法：
$ cat tst.awk
BEGIN { FS="[][]" }
{
split($2,t,/[/: ]/)
t[2] = (index("JanFebMarAprMayJunJulAugSepOctNovDec",t[2])+2)/3
epochMins = int(mktime(t[3] " " t[2] " " t[1] " " t[4] " " t[5] " 0") / 60)
if (NR == 1) {
begMins = epochMins
}
endMins = epochMins
cnt[epochMins]++
}
END {
for (epochMins = begMins; epochMins <= endMins; epochMins+=interval) {
begTime = strftime("%H:%M",epochMins*60)
endTime = strftime("%H:%M",(epochMins+interval)*60)
tot = 0
begInterval = epochMins
endInterval = epochMins + interval - 1
for (i=begInterval; i<=endInterval; i++) {
tot += cnt[i]
}
print begTime, endTime ORS tot
}
}
.
$ awk -v interval=1 -f tst.awk file
09:01 09:02
6
09:02 09:03
0
09:03 09:04
0
09:04 09:05
0
09:05 09:06
0
09:06 09:07
8
09:07 09:08
0
09:08 09:09
0
09:09 09:10
0
09:10 09:11
0
09:11 09:12
14
09:12 09:13
0
09:13 09:14
0
09:14 09:15
4
.
$ awk -v interval=2 -f tst.awk file
09:01 09:03
6
09:03 09:05
0
09:05 09:07
8
09:07 09:09
0
09:09 09:11
0
09:11 09:13
14
09:13 09:15
4
.
$ awk -v interval=5 -f tst.awk file
09:01 09:06
6
09:06 09:11
8
09:11 09:16
18
好吧，它有点笨拙，在文件的每一行上运行date，read两次(呃(，但请尝试这样的事情：
#! /usr/bin/env bash
declare -i interval=${1:- 1} timestamp=0 elapsed=0 last=0 cnt=0
declare H M lh lm # reporting vars
echo "Counting hits per ${interval}m interval"
interval=$(( interval * 60 )) # convert to seconds
while IFS="][" read x d x && [[ -n "$d" ]] # pull just the the timestamp
do IFS="$IFS/:" read day mon year h m s x <<< "$d" # parse components
timestamp="$( date -d "$mon $day $year $h:$m:$s" +'%s' )" # reformat to epoch secs
if (( last ))
then elapsed="$(( timestamp - last ))" # check elapsed time since last
if (( elapsed > interval ))
then printf "$H:$M - $lh:$lmn$cntn"
cnt=1
last=$timestamp
H=$h; M=$m;
lh=$h; lm=$m;
else cnt+=1
lh=$h; lm=$m;
fi
else last=$timestamp # assure initialized
H=$h; M=$m;
lh=$h; lm=$m;
cnt=1
fi
done < "$yourLogFile" # you'll need to set this
printf "$H:$M - $lh:$lmn$cntn"
当分钟发生变化时，它会检查是否已超过间隔，如果有，则报告计数。
$: ./tst
Counting hits per 1m interval
09:01 - 09:01
6
09:06 - 09:06
8
09:11 - 09:11
14
09:14 - 09:14
4
$: ./tst 3
Counting hits per 3m interval
09:01 - 09:01
6
09:06 - 09:06
8
09:11 - 09:14
18
$: ./tst 7
Counting hits per 7m interval
09:01 - 09:06
14
09:11 - 09:14
18
这应该让你开始
。一个粗略的全awk版本——
#! /usr/bin/env bash
awk -v interval=${1:- 1} '
BEGIN { last = 0; cnt = 0;
printf "Counting hits per %sm intervaln", interval
interval = interval * 60
}
/:/ {
split( $0, tmp, "[" )
split( tmp[2], t2, "]" )
split( t2[1], tmp, ":" ) # tmp now date, H, M, S, tz
H = tmp[2]; M = tmp[3]
split( tmp[1], dtmp, "/" ) # dtmp now dd, Mon, yyyy
switch ( dtmp[2] ) {
case "Jan": Mon = "01"; break; case "Feb": Mon = "02"; break; case "Mar": Mon = "03"; break;
case "Apr": Mon = "04"; break; case "May": Mon = "05"; break; case "Jun": Mon = "06"; break;
case "Jul": Mon = "07"; break; case "Aug": Mon = "08"; break; case "Sep": Mon = "09"; break;
case "Oct": Mon = "10"; break; case "Nov": Mon = "11"; break; case "Dec": Mon = "12"; break;
}
tstr = sprintf( "%s %s %s %s %s %s", dtmp[3], Mon, dtmp[1], H, M, "00" )
epoch = mktime( tstr )
if ( last ) {
elapsed = epoch - last
if ( elapsed > interval ) {
printf "%s:%s - %s:%sn%sn", h, m, lh, lm, cnt
cnt = 1; last = epoch; h=H; m=M; lh=H; lm=M;
} else { cnt+=1; lh=H; lm=M; }
} else { last = epoch; h=H; m=M; lh=H; lm=M; cnt=1; }
next;
}
END { printf "%s:%s - %s:%sn%sn", h, m, lh, lm, cnt }
' ${2:-defaultFileName}
对于大文件输入，这应该更有效率。