#!/usr/bin/env python
# -*- coding: utf-8 -*-

"""
access_analyzer.py

Apache httpd access log または抽出済みアクセスログから、
公開資料ごとのアクセス数を集計して Excel に出力する。

対応する入力形式:

1) Apache combined/common に近い形式
   59.132.62.132 - - [05/Apr/2026:03:57:54 +0900] "GET /path/file.html HTTP/1.1" 200 1234 "..." "UA"

2) 既存 access_filter.py で保存した簡略形式
   2026-04-05 03:57:54+09:00 59.132.62.132 /D2MatE/0tutorial/2025/xxx.mp3

出力:
- summary
    URL/ファイルごとの集計
- daily
    日別・URL別集計
- matched_logs
    マッチしたアクセス明細

例:

python access_analyzer.py ^
  --log ./saved_logs ^
  --url-pattern "/D2MatE/0tutorial/2025/*" ^
  --from 2026-04-01 ^
  --to 2026-04-30 ^
  --out tutorial_access.xlsx

python access_analyzer.py --log "access_list.txt" --url-pattern "*.mp4" "*.pptx" "*.docx" --exclude-bot 0 
python access_analyzer.py --log "access_list.txt" --url-pattern "*.mp4" "*.pptx" "*.docx" --from 2025-04-01 --to 2026-03-30  --exclude-bot 0
"""

import argparse
import fnmatch
import glob
import gzip
import re
import sys
from dataclasses import dataclass
from datetime import datetime, timedelta, timezone
from pathlib import Path
from urllib.parse import unquote, urlsplit

try:
    import pandas as pd
except Exception:
    import traceback
    print("ERROR: pandas is required.")
    traceback.print_exc()
    print("\nInstall example:")
    print("  pip install pandas openpyxl")
    input("\nPress ENTER to terminate>>\n")
    sys.exit(1)


# Apache combined/common 形式の先頭部分を読む
APACHE_LOG_RE = re.compile(
    r'(?P<ip>\S+) \S+ \S+ '
    r'\[(?P<time>[^\]]+)\] '
    r'"(?P<method>\S+)\s+(?P<url>\S+)\s+(?P<proto>[^"]+)" '
    r'(?P<status>\d{3}) '
    r'(?P<size>\S+)'
    r'(?:\s+"(?P<referer>[^"]*)"\s+"(?P<agent>[^"]*)")?'
)

# access_filter.py が出力していた簡略形式
# 2026-04-05 03:57:54+09:00 59.132.62.132 /path/to/file
SIMPLE_LOG_RE = re.compile(
    r'(?P<date>\d{4}-\d{2}-\d{2})\s+'
    r'(?P<time>\d{2}:\d{2}:\d{2}(?:[+-]\d{2}:\d{2})?)\s+'
    r'(?P<ip>\S+)\s+'
    r'(?P<url>\S.*)$'
)


@dataclass
class AccessEntry:
    time: datetime
    ip: str
    url: str
    path: str
    key: str
    status: int | None = None
    user_agent: str | None = None
    source_log: str | None = None


def parse_args():
    parser = argparse.ArgumentParser(
        description="Apache httpd log / 抽出済みlogから、資料URLごとのアクセス数をExcel集計する"
    )

    parser.add_argument(
        "--log",
        required=True,
        help="解析対象ログファイル、ディレクトリ、またはglobパターン"
    )

    parser.add_argument(
        "--url-pattern",
        "--url-patterns",
        nargs="+",
        required=True,
        help='URLワイルドカード。複数指定可。例: "/D2MatE/0tutorial/2025/*" "*.pdf"'
    )

    parser.add_argument(
        "--out",
        default="access_summary.xlsx",
        help="出力Excelファイル名"
    )

    parser.add_argument(
        "--from",
        dest="date_from",
        default=None,
        help="開始日 YYYY-MM-DD。この日の00:00:00以降を含む"
    )

    parser.add_argument(
        "--to",
        dest="date_to",
        default=None,
        help="終了日 YYYY-MM-DD。この日の23:59:59までを含む"
    )

    parser.add_argument(
        "--ip",
        default=None,
        help="IPフィルタ。ワイルドカード可。例: 133.*.*.*"
    )

    parser.add_argument(
        "--status",
        nargs="*",
        type=int,
        default=None,
        help="集計対象status。例: --status 200 206 304。簡略ログでは無視される"
    )

    parser.add_argument(
        "--exclude-bot",
        type=int,
        choices=[0, 1],
        default=1,
        help="botらしいUser-Agentを除外する。1=除外, 0=除外しない。簡略ログでは判定不可"
    )

    parser.add_argument(
        "--strip-query",
        type=int,
        choices=[0, 1],
        default=1,
        help="URLの?以降を除去して集計する。1=除去, 0=残す"
    )

    parser.add_argument(
        "--key-mode",
        choices=["path", "filename"],
        default="path",
        help="集計キー。path=URLパス全体, filename=ファイル名のみ"
    )

    parser.add_argument(
        "--save-matched",
        type=int,
        choices=[0, 1],
        default=1,
        help="matched_logsシートを保存する。1=保存, 0=保存しない"
    )

    parser.add_argument(
        "--verbose",
        type=int,
        choices=[0, 1],
        default=1,
        help="処理状況を表示する。1=表示, 0=非表示"
    )

    return parser.parse_args()


def parse_date_range(date_from_s, date_to_s):
    date_from = None
    date_to = None

    if date_from_s:
        date_from = datetime.strptime(date_from_s, "%Y-%m-%d").replace(
            tzinfo=timezone(timedelta(hours=9))
        )

    if date_to_s:
        date_to = datetime.strptime(date_to_s, "%Y-%m-%d").replace(
            tzinfo=timezone(timedelta(hours=9))
        )
        date_to = date_to + timedelta(days=1) - timedelta(microseconds=1)

    return date_from, date_to


def collect_log_files(log_path: str):
    p = Path(log_path)

    if p.is_file():
        return [p]

    if p.is_dir():
        patterns = [
            "*.log",
            "*access_log*",
            "*.gz",
            "*.txt",
        ]
        files = []
        seen = set()

        for pat in patterns:
            for f in p.glob(pat):
                if f not in seen and f.is_file():
                    files.append(f)
                    seen.add(f)

        return sorted(files)

    files = [Path(f) for f in glob.glob(log_path)]
    files = [f for f in files if f.is_file()]

    if files:
        return sorted(files)

    raise ValueError(f"ログファイル、ディレクトリ、globパターンが見つかりません: {log_path}")


def open_log_file(path: Path):
    if path.suffix == ".gz":
        return gzip.open(path, "rt", encoding="utf-8", errors="ignore")
    return open(path, "r", encoding="utf-8", errors="ignore")


def normalize_url(url: str, strip_query: bool = True):
    decoded = unquote(url)

    if strip_query:
        parsed = urlsplit(decoded)
        path = parsed.path
    else:
        path = decoded

    return decoded, path


def make_key(path: str, key_mode: str):
    if key_mode == "filename":
        name = Path(path).name
        return name if name else path
    return path


def is_bot_user_agent(user_agent: str | None):
    if not user_agent:
        return False

    ua = user_agent.lower()

    bot_keywords = [
        "bot",
        "crawl",
        "spider",
        "slurp",
        "mediapartners",
        "googlebot",
        "bingbot",
        "duckduckbot",
        "baiduspider",
        "yandex",
        "semrush",
        "ahrefs",
        "mj12bot",
        "petalbot",
        "python-requests",
        "curl",
        "wget",
    ]

    return any(k in ua for k in bot_keywords)


def parse_apache_line(line: str, strip_query: bool, key_mode: str, source_log: str):
    m = APACHE_LOG_RE.match(line)
    if not m:
        return None

    try:
        t = datetime.strptime(m.group("time"), "%d/%b/%Y:%H:%M:%S %z")
    except Exception:
        return None

    url = m.group("url")
    decoded_url, path = normalize_url(url, strip_query=strip_query)

    return AccessEntry(
        time=t,
        ip=m.group("ip"),
        url=decoded_url,
        path=path,
        key=make_key(path, key_mode),
        status=int(m.group("status")),
        user_agent=m.group("agent"),
        source_log=source_log,
    )


def parse_simple_line(line: str, strip_query: bool, key_mode: str, source_log: str):
    m = SIMPLE_LOG_RE.match(line)
    if not m:
        return None

    dt_s = f"{m.group('date')} {m.group('time')}"

    try:
        # 例: 2026-04-05 03:57:54+09:00
        t = datetime.fromisoformat(dt_s)
    except Exception:
        try:
            # timezoneなしの場合はJST扱い
            t = datetime.strptime(dt_s, "%Y-%m-%d %H:%M:%S").replace(
                tzinfo=timezone(timedelta(hours=9))
            )
        except Exception:
            return None

    url = m.group("url").strip()
    decoded_url, path = normalize_url(url, strip_query=strip_query)

    return AccessEntry(
        time=t,
        ip=m.group("ip"),
        url=decoded_url,
        path=path,
        key=make_key(path, key_mode),
        status=None,
        user_agent=None,
        source_log=source_log,
    )


def parse_line(line: str, strip_query: bool, key_mode: str, source_log: str):
    line = line.strip()
    if not line:
        return None

    entry = parse_apache_line(line, strip_query, key_mode, source_log)
    if entry is not None:
        return entry

    entry = parse_simple_line(line, strip_query, key_mode, source_log)
    if entry is not None:
        return entry

    return None


def match_url(path: str, decoded_url: str, patterns: list[str]):
    for pat in patterns:
        if fnmatch.fnmatch(path, pat):
            return True
        if fnmatch.fnmatch(decoded_url, pat):
            return True
    return False


def in_date_range(t: datetime, date_from, date_to):
    if date_from is not None and t < date_from:
        return False
    if date_to is not None and t > date_to:
        return False
    return True


def collect_entries(files, args, date_from, date_to):
    entries = []
    n_lines = 0
    n_parsed = 0

    for log_file in files:
        if args.verbose:
            print(f"Reading: {log_file}")

        with open_log_file(log_file) as fp:
            for line in fp:
                n_lines += 1

                entry = parse_line(
                    line,
                    strip_query=bool(args.strip_query),
                    key_mode=args.key_mode,
                    source_log=str(log_file),
                )

                if entry is None:
                    continue

                n_parsed += 1

                if not in_date_range(entry.time, date_from, date_to):
                    continue

                if args.ip and not fnmatch.fnmatch(entry.ip, args.ip):
                    continue

                if args.status is not None and entry.status is not None:
                    if entry.status not in args.status:
                        continue

                if args.exclude_bot and is_bot_user_agent(entry.user_agent):
                    continue

                if not match_url(entry.path, entry.url, args.url_pattern):
                    continue

                entries.append(entry)

    return entries, n_lines, n_parsed


def build_dataframes(entries: list[AccessEntry]):
    rows = []

    for e in entries:
        rows.append({
            "datetime": e.time.isoformat(),
            "date": e.time.date().isoformat(),
            "ip": e.ip,
            "url": e.url,
            "path": e.path,
            "key": e.key,
            "status": e.status,
            "user_agent": e.user_agent,
            "source_log": e.source_log,
        })

    df = pd.DataFrame(rows)

    if df.empty:
        summary = pd.DataFrame(columns=[
            "key",
            "total_access",
            "unique_ip_all_period",
            "sum_daily_unique_ip",
            "first_access",
            "last_access",
        ])

        daily = pd.DataFrame(columns=[
            "date",
            "key",
            "total_access",
            "unique_ip",
        ])

        return summary, daily, df

    daily = (
        df.groupby(["date", "key"], dropna=False)
          .agg(
              total_access=("ip", "size"),
              unique_ip=("ip", "nunique"),
          )
          .reset_index()
          .sort_values(["key", "date"])
    )

    daily_unique_sum = (
        daily.groupby("key", dropna=False)["unique_ip"]
             .sum()
             .reset_index()
             .rename(columns={"unique_ip": "sum_daily_unique_ip"})
    )

    summary_base = (
        df.groupby("key", dropna=False)
          .agg(
              total_access=("ip", "size"),
              unique_ip_all_period=("ip", "nunique"),
              first_access=("datetime", "min"),
              last_access=("datetime", "max"),
          )
          .reset_index()
    )

    summary = summary_base.merge(daily_unique_sum, on="key", how="left")

    summary = summary[
        [
            "key",
            "total_access",
            "unique_ip_all_period",
            "sum_daily_unique_ip",
            "first_access",
            "last_access",
        ]
    ].sort_values("total_access", ascending=False)

    df = df.sort_values("datetime", ascending=False)

    return summary, daily, df


def write_excel(out_path: str, summary: pd.DataFrame, daily: pd.DataFrame, matched: pd.DataFrame, save_matched: bool):
    out = Path(out_path)

    with pd.ExcelWriter(out, engine="openpyxl") as writer:
        summary.to_excel(writer, sheet_name="summary", index=False)
        daily.to_excel(writer, sheet_name="daily", index=False)

        if save_matched:
            matched.to_excel(writer, sheet_name="matched_logs", index=False)

    return out


def main():
    args = parse_args()

    print("=== access_analyzer settings ===")
    print(f"log          = {args.log}")
    print(f"url_pattern  = {args.url_pattern}")
    print(f"out          = {args.out}")
    print(f"date_from    = {args.date_from}")
    print(f"date_to      = {args.date_to}")
    print(f"ip           = {args.ip}")
    print(f"status       = {args.status}")
    print(f"exclude_bot  = {args.exclude_bot}")
    print(f"strip_query  = {args.strip_query}")
    print(f"key_mode     = {args.key_mode}")

    date_from, date_to = parse_date_range(args.date_from, args.date_to)

    files = collect_log_files(args.log)

    print("\n=== log files ===")
    for f in files:
        print(f"  {f}")

    entries, n_lines, n_parsed = collect_entries(files, args, date_from, date_to)

    summary, daily, matched = build_dataframes(entries)

    out = write_excel(
        args.out,
        summary,
        daily,
        matched,
        save_matched=bool(args.save_matched),
    )

    print("\n=== summary ===")
    print(f"read lines        : {n_lines}")
    print(f"parsed lines      : {n_parsed}")
    print(f"matched accesses  : {len(entries)}")
    print(f"summary rows      : {len(summary)}")
    print(f"daily rows        : {len(daily)}")
    print(f"output            : {out}")

    if not summary.empty:
        print("\n=== top accessed files ===")
        print(summary.head(20).to_string(index=False))


if __name__ == "__main__":
    main()