Extracting API Requests Counts From Nginx Access Log Per week

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42

#!/usr/bin/env bash
# Dec 12, 2017 Tue +0800

# variable setting
log_path='/data/log/api.deerbus.com_access.log'
extract_log_path=$(mktemp -t XXXXX.log)

# get date range
# last week monday
start_date=$(date --date='last monday 7 days ago' +'%d/%b/%Y')
# this monday
stop_date=$(date --date='last monday' +'%d/%b/%Y')

# increase file descriptor value
file_descriptor_val=${file_descriptor_val:-1024}
file_descriptor_val=$(ulimit -n)
ulimit -n 655360

# extract log in specified date range (a week)
start_line_no=1
[[ $(grep -c "${start_date}" "${log_path}") -gt 0 ]] && start_line_no="${start_date}"

sed -r -n '/\['"${start_line_no////\\/}"'/,/\['"${stop_date////\\/}"'/{/'"${stop_date////\\/}"'/d;/^$/d;[email protected][[:space:]]@@g;[email protected][[:space:]]*\+[[:digit:]][email protected]@g;[email protected]\"[[:upper:]]+ ([^[:space:]]*) .*@\[email protected];[email protected](\[|\])@@g;[email protected] @|@g;[email protected]([[:digit:]]{4}):@\1 @g;[email protected]([[:digit:]]{1,2})/([[:alpha:]]+)/([[:digit:]]{4})@\1 \2 \[email protected];p}' "${log_path}"  > "${extract_log_path}"

# output sample
# 58.38.100.200|10 Dec 2017 23:59:57|/buspay/buildPayParams.do

# extract specific info according differect degrees
# 1 - all api per seconds
awk -F\| 'BEGIN{OFS="|"}{arr[$2]++}END{for (i in arr) print i,arr[i]}' "${extract_log_path}" | sort -b -f -t"|" -k 2nr,2 | awk -F\| '{if($2>50){"date --date=\""$1"\" +\"%F %T\"" | getline a;printf("%s|%s\n",a,$2)}}' > /tmp/total_counts_per_second.txt

# 2 - per api per seconds
awk -F\| 'BEGIN{OFS="|"}{arr[$NF,$2]++}END{for (i in arr) {split(i,j,SUBSEP);print j[2],j[1],arr[i]}}' "${extract_log_path}" | sort -b -f -t"|" -k 3nr,3 -k 1nr,1 | awk -F\| '{if($3>10){"date --date=\""$1"\" +\"%F %T\"" | getline a;printf("%s|%s|%s\n",a,$2,$3)}}' > /tmp/total_counts_per_second.txt

# 3 - total counts for specific api
awk -F\| 'BEGIN{OFS="|"}{arr[$3]++}END{for (i in arr) print i,arr[i]}' "${extract_log_path}" | sort -b -f -t"|" -k 2nr,2 | awk -F\| '{if($2>50) print}' > /tmp/total_counts_per_api.txt


[[ -f "${extract_log_path}" ]] && /bin/rm -f "${extract_log_path}"
ulimit -n "${file_descriptor_val}"

# Script End

1

awk -F\| 'BEGIN{OFS="|"}{"date --date=\""$(NF-1)"\" +\"%F %T\"" | getline a;arr[a]++}END{for (i in arr) print i,arr[i]}' "${extract_log_path}" | sort -b -f -t"|" -k 2r,2 > /tmp/total_counts_per_second.txt

1

awk -F\| 'BEGIN{OFS="|"}{arr[$2]++}END{for (i in arr) print i,arr[i]}' "${extract_log_path}" | sort -b -f -t"|" -k 2nr,2 | awk -F\| '{if($2>50){"date --date=\""$1"\" +\"%F %T\"" | getline a;printf("%s|%s\n",a,$2)}}' > /tmp/total_counts_per_second.txt