Using GoAccess To Analyze Web Server Log On GNU/Linux

1

114.114.114.114 - - [07/Sep/2016:18:23:22 +0800] "POST /bususer/flybusLogin.do HTTP/1.1" 200 0 "-" "okhttp/3.1.2"

1

$remote_addr - $remote_user [$time_local] "$request" $status $body_bytes_sent "$http_referer" "$http_user_agent"

1
2
3
4
5
6

#CentOS
sudo yum install -y epel-release
sudo yum install -y gcc make ncurses ncurses-devel geoip geoip-devel tokyocabinet tokyocabinet-devel

#Debian
sudo apt-get install libncursesw5-dev libgeoip-dev libtokyocabinet-dev

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19

# 創建目錄 /usr/local/bin
/usr/bin/mkdir -p '/usr/local/bin'
# 將可執行文件 goaccess 複製到該目錄中
/usr/bin/install -c goaccess '/usr/local/bin'

# 創建目錄 /usr/local/etc
/usr/bin/mkdir -p '/usr/local/etc'
# 將文件 goaccess.conf 複製到該目錄中並賦予644權限
/usr/bin/install -c -m 644 config/goaccess.conf '/usr/local/etc'

# 創建目錄 /usr/local/share/doc/goaccess
/usr/bin/mkdir -p '/usr/local/share/doc/goaccess'
# 將相關文件複製到該目錄中
/usr/bin/install -c -m 644 resources/tpls.html resources/css/app.css resources/css/bootstrap.min.css resources/css/fa.min.css resources/js/app.js resources/js/charts.js resources/js/d3.v3.min.js resources/js/hogan.min.js '/usr/local/share/doc/goaccess'

# 創建目錄 /usr/local/share/man/man1
/usr/bin/mkdir -p '/usr/local/share/man/man1'
# 將文件 goaccess.1 複製到該目錄中並賦予644權限
/usr/bin/install -c -m 644 goaccess.1 '/usr/local/share/man/man1'

1

sudo make uninstall

1
2
3
4

cd '/usr/local/bin' && rm -f goaccess
cd '/usr/local/etc' && rm -f goaccess.conf
cd '/usr/local/share/doc/goaccess' && rm -f tpls.html app.css bootstrap.min.css fa.min.css app.js charts.js d3.v3.min.js hogan.min.js
cd '/usr/local/share/man/man1' && rm -f goaccess.1

1
2
3
4
5

# 數據存儲在內存中
--enable-tcb=memhash

# 數據存儲在磁盤中
--enable-tcb=btree

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24

# Way 1: Directly Download

## 通過curl
curl -# -o /tmp/goaccess.tar.gz http://tar.goaccess.io/goaccess-1.0.2.tar.gz
cd /tmp && tar -xzf goaccess.tar.gz -C /tmp
## 通過wget
# wget -q http://tar.goaccess.io/goaccess-1.0.2.tar.gz -P /tmp
# cd /tmp && tar -xzf goaccess-1.0.2.tar.gz -C /tmp

mv goaccess-1.0.2 goaccess
cd ./goaccess


# Way 2: Build from GitHub (Development)
git clone https://github.com/allinurl/goaccess.git /tmp #下載到臨時目錄中
cd /tmp/goaccess
autoreconf -fiv


# compile
# ./configure --prefix=/usr/local --sysconfdir=/etc/goaccess --enable-geoip --enable-utf8
./configure --prefix=/usr/local --sysconfdir=/etc/goaccess --enable-geoip --enable-utf8 --enable-tcb=memhash --with-getline
make -j 4   #Specifies the number of jobs (commands) to run simultaneously.
sudo make install

1
2
3
4
5
6
7
8
9

[[email protected] ~]$ which goaccess
/usr/local/bin/goaccess
[[email protected] ~]$ ls /etc/goaccess/goaccess.conf
/etc/goaccess/goaccess.conf
[[email protected] ~]$ goaccess -V
GoAccess - 1.0.2.
For more details visit: http://goaccess.io
Copyright (C) 2009-2016 by Gerardo Orellana
[[email protected] ~]$

1
2

sudo yum install -y epel-release
sudo yum install -y goaccess #在epel倉庫中

1
2
3
4

echo "deb https://deb.goaccess.io/ $(lsb_release -cs) main" | sudo tee -a /etc/apt/sources.list.d/goaccess.list
wget -O - https://deb.goaccess.io/gnugpg.key | sudo apt-key add -
sudo apt-get update
sudo apt-get install goaccess

1

goaccess -f /tmp/api_access.log

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19
20

+-----------------------------------------------------------+
| Log Format Configuration                                  |
| [SPACE] to toggle - [ENTER] to proceed                    |
|                                                           |
| [ ] NCSA Combined Log Format                              |
| [ ] NCSA Combined Log Format with Virtual Host            |
| [ ] Common Log Format (CLF)                               |
| [ ] Common Log Format (CLF) with Virtual Host             |
| [ ] W3C                                                   |
| [ ] Squid Native Format                                   |
|                                                           |
| Log Format - [c] to add/edit format                       |
|                                                           |
|                                                           |
| Date Format - [d] to add/edit format                      |
|                                                           |
|                                                           |
| Time Format - [t] to add/edit format                      |
|                                                           |
+-----------------------------------------------------------+

1
2
3
4
5
6
7
8

# Apache/NGINX's log formats below.
time-format %H:%M:%S

# Apache/NGINX's log formats below.
date-format %d/%b/%Y

# NCSA Combined Log Format
log-format %h %^[%d:%t %^] "%r" %s %b "%R" "%u"

1

goaccess -f /tmp/api_access.log

1
2
3
4
5
6

sed -n '/02\/Sep\/2016/,/19\/Sep\/2016:12/ p' /tmp/api_access.log > /tmp/access.log

goaccess -f /tmp/access.log -o ~/Desktop/report.html
# goaccess -f /tmp/access.log > ~/Desktop/report.html

google-chrome ~/Desktop/report.html

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19

create database if not exists log_analysis
    default character set=utf8
    default collate=utf8_general_ci;

use log_analysis;

create table if not exists log_analysis (
    id int unsigned not null auto_increment primary key comment '自增id',
    ip char(20) not null comment 'ip地址',
    access_time timestamp not null comment '訪問接口時間,格式 YYYY-MM-DD HH:MM:SS',
    method enum('POST','GET') default 'POST' comment 'api訪問methond',
    api_url char(60) not null comment 'api 地址',
    response_code char(3) not null comment '響應碼',
    key `log_ip` (`ip`),
    key `log_access_time` (`access_time`),
    key `log_method` (`method`),
    key `log_api_url` (`api_url`),
    key `log_response_code` (`response_code`)
)engine=innodb default charset=utf8 collate=utf8_general_ci comment '接口訪問數據統計表';

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45

#!/bin/bash

log_path='/tmp/analysis.txt'
# log_path='/tmp/test.txt'
target_file=`mktemp -t tempXXXXX.txt`
# 數據格式 116.228.89.242 - - [07/Sep/2016:18:23:22 +0800] "POST /bususer/flybusLogin.do HTTP/1.1" 200 0 "-" "okhttp/3.1.2"

while read line; do
    ip=`echo "$line" | awk '{print $1}'`
    api_url=`echo "$line" | awk '{print $7}'`
    response_code=`echo "$line" | awk '{print $9}'`

    # 日期時間處理 access_time
    timetemp=`echo "$line" | awk '{print $4}'`
    timetemp=${timetemp#*[}
    hms=${timetemp#*:}
    ymdtemp=${timetemp%%:*}
    ymdtemp=${ymdtemp/Sep/09}
    ymd=$(echo $ymdtemp | awk -v FS='/' '{printf("%s-%s-%s",$3,$2,$1)}')
    access_time=$ymd' '$hms

    # method處理 method
    methodtemp=`echo "$line" | awk '{print $6}'`
    method=${methodtemp#*\"}

    mysql -e "insert into log_analysis.log_analysis set ip='$ip',access_time='$access_time',method='$method',api_url='$api_url',response_code='$response_code';"

    # echo "$ymd $hms $hm $api_url $ip $response_code $method" >> "$target_file"

    unset ip
    unset api_url
    unset response_code
    unset timetemp
    unset hms
    unset ymdtemp
    unset ymd
    unset access_time
    unset methodtemp
    unset method
    # echo "$response_code"

    # awk '{print $1}' "$line"
done<"$log_path"

# End Script

1
2
3

mysql -D log_analysis -Bse "select api_url as 'API',count(id) as Frequency from log_analysis group by api_url order by Frequency desc;" | awk '{printf("%s —— %s\n",$1,$2)}'

mysql -D log_analysis -Bse "select api_url, count(id) as num, date_format(access_time, '%Y年%m月%d日 %H時%i分') as accessTime from log_analysis group by api_url,date_format(access_time, '%Y年%m月%d日 %H時%i分') order by num desc limit 20;" | awk '{printf("%s —— %s —— %s\n",$1,$2,$3)}'