import sys
import re
import requests
from urllib.parse import urljoin, urlparse, urlunparse
from xml.dom.minidom import parseString
import xml.etree.ElementTree as ET
from datetime import datetime
import charset_normalizer


RootURL = "http://d2mate.mdxes.iir.isct.ac.jp/D2MatE/D2MatE_programs.html?page=statistcs"
outpath = "sitemap.xml"

argv = sys.argv
nargs = len(argv)
if nargs > 1:
    RootURL = argv[1]
if nargs > 2:
    outpath = argv[2]

base_domain = urlparse(RootURL).netloc

print()
print("Crawling and generating XML Sitemap with metadata...")
print()
print(f"usage: python {argv[0]} [RootURL] [OutputFile]")
print()
print(f"RootURL: {RootURL}")
print(f"Output File: {outpath}")
print(f"Base Domain: {base_domain}")
print()

def clean_html_content(html_content):
    """
    改行文字を削除し、前後のスペースをトリムする関数
    """
    return " ".join(html_content.split())

def get_page_title(html_content):
    """
    HTMLコンテンツ内から<title>を抽出する関数
    """
    title_match = re.search(r"<title>(.*?)</title>", html_content, re.IGNORECASE)
    if title_match:
        return title_match.group(1).strip()
    return "No Title"

def is_target_url(url):
    """
    URLがHTMLファイル（*.html）またはファイル名なし（ディレクトリパス）の場合にTrueを返す
    """
    parsed_url = urlparse(url)
    path = parsed_url.path
    return path.endswith(".html") or path.endswith(".shtml") or path.endswith("/")

def normalize_url(url):
    """
    URLを標準化する（クエリパラメータを保持する）
    """
    parsed_url = urlparse(url)
    normalized = urlunparse(parsed_url._replace(fragment=""))  # フラグメントを削除
    return normalized

def find_urls_in_html(html_content, base_url):
    """
    HTMLコンテンツからリンクを抽出し、FQDNを生成する関数
    """
    urls = set()
    pattern = re.compile(r"url:\s*[\"']([^\"']+)[\"']|href=[\"']([^\"']+)[\"']|src=[\"']([^\"']+)[\"']")
    matches = pattern.findall(html_content)

    for match in matches:
        url = match[0] or match[1] or match[2]
        if url.startswith("/"):
            full_url = urljoin(base_url, url)
        elif not url.startswith("http"):
            full_url = urljoin(base_url, url)
        else:
            full_url = url
        if is_target_url(full_url):
            urls.add(full_url)
    return urls

def detect_encoding(response_content):
    """
    文字コードを判別する関数
    """
    result = charset_normalizer.detect(response_content)
    return result['encoding']

def crawl_recursive(start_url, visited, base_domain, parent = ""):
    """
    再帰的にリンクをたどる関数（クエリパラメータ含めた情報の取得対応）
    """
    normalized_url = normalize_url(start_url)

    if normalized_url in visited:
        return {}

    print(f"Crawling: {start_url}")
    visited.add(normalized_url)

    try:
        response = requests.get(start_url)
        if response.status_code != 200:
            print(f"Failed to fetch: {start_url} in {parent}")
            return {}

        # 文字コード判別
        encoding = detect_encoding(response.content)
        decoded_content = response.content.decode(encoding)
        print(f" Encoding: {encoding}")

        # Last-Modifiedヘッダーを取得
        lastmod = response.headers.get("Last-Modified")
        if lastmod:
            lastmod = datetime.strptime(lastmod, "%a, %d %b %Y %H:%M:%S %Z").strftime("%Y-%m-%d")
            print(f" lastmod: {lastmod}")
        else:
            lastmod = None

    except Exception as e:
        print(f"Error fetching {start_url}: {e}")
        return {}

    html_content = clean_html_content(decoded_content)
    title = get_page_title(html_content)
    print(f" title: {title}")

    found_urls = find_urls_in_html(html_content, start_url)

    filtered_urls = {url for url in found_urls if urlparse(url).netloc == base_domain}

    all_urls = {start_url: {"title": title, "lastmod": lastmod}}
    for url in filtered_urls:
        all_urls.update(crawl_recursive(url, visited, base_domain, parent = start_url))

    return all_urls

def generate_xml_sitemap_with_metadata(urls_with_metadata, output_file=None):
    """
    URLとメタデータ（タイトル、Last-Modified）から整形済みXMLサイトマップを生成する関数
    """
    
    if output_file is None:
        output_file = outpath;
    
    urlset = ET.Element("urlset", xmlns="http://www.sitemaps.org/schemas/sitemap/0.9")
    for url, metadata in sorted(urls_with_metadata.items()):
        url_element = ET.SubElement(urlset, "url")
        loc = ET.SubElement(url_element, "loc")
        loc.text = url

        title = metadata.get("title", "No Title")
        title_element = ET.SubElement(url_element, "title")
        title_element.text = title

        lastmod = metadata.get("lastmod")
        if lastmod:
            lastmod_element = ET.SubElement(url_element, "lastmod")
            lastmod_element.text = lastmod

        changefreq = ET.SubElement(url_element, "changefreq")
        changefreq.text = "monthly"

        priority = ET.SubElement(url_element, "priority")
        priority.text = "1.0"

    raw_xml = ET.tostring(urlset, encoding="utf-8")
    parsed_xml = parseString(raw_xml)
    pretty_xml = parsed_xml.toprettyxml(indent="  ")

    with open(output_file, "w", encoding="utf-8") as f:
        f.write(pretty_xml)

def main():
    visited = set()

    print(f"Starting crawl from: {RootURL}")
    all_urls_with_metadata = crawl_recursive(RootURL, visited, base_domain)

    print(f"Found total URLs: {len(all_urls_with_metadata)}")
    for idx, (url, metadata) in enumerate(sorted(all_urls_with_metadata.items()), start=1):
        print(f" [{idx}] {url} - Title: {metadata['title']} - LastMod: {metadata['lastmod']}")

    generate_xml_sitemap_with_metadata(all_urls_with_metadata)
    print("XML Sitemap with metadata has been saved as '{outpath}'")

if __name__ == "__main__":
    main()