diff --git a/mitmproxy/INSTALL.md b/mitmproxy/INSTALL.md new file mode 100644 index 0000000..650b025 --- /dev/null +++ b/mitmproxy/INSTALL.md @@ -0,0 +1,312 @@ +# Установка и запуск mitmproxy на VPS + +## 1. Установка mitmproxy + +### Вариант A: pip (рекомендуется) + +```bash +# Установить pip +apt-get update && apt-get install -y python3-pip + +# Установить mitmproxy +pip3 install mitmproxy + +# Проверка +mitmdump --version +``` + +### Вариант B: Docker (если pip не работает) + +```bash +# Установить Docker +apt-get install -y docker.io + +# Запустить в контейнере +docker run --rm -it -p 8080:8080 mitmproxy/mitmproxy \ + mitmdump -p 8080 --mode regular +``` + +### Вариант C: apt (старая версия) + +```bash +apt-get install -y mitmproxy +``` + +--- + +## 2. Настройка firewall + +```bash +# Открыть порт 8080 +ufw allow 8080/tcp + +# Проверить +ufw status +``` + +--- + +## 3. Генерация сертификата + +```bash +# Первый запуск создаст сертификат +mitmproxy --version + +# Сертификаты будут в ~/.mitmproxy/ +ls -la ~/.mitmproxy/ +``` + +--- + +## 4. Запуск с авторизацией + +### Быстрый старт + +```bash +cd /root/kwork-parser/mitmproxy +chmod +x start.sh +./start.sh +``` + +### Вручную + +```bash +mitmdump \ + -s /root/kwork-parser/mitmproxy/auth.py \ + --set auth_user=claw \ + --set auth_pass="KworkParser2026!" \ + -p 8080 \ + --mode regular \ + -w /root/kwork-parser/mitmproxy/flows.mitm +``` + +### Как systemd сервис + +```bash +# Установить сервис +cp /root/kwork-parser/mitmproxy/mitmproxy.service /etc/systemd/system/ + +# Запустить +systemctl daemon-reload +systemctl enable mitmproxy +systemctl start mitmproxy + +# Проверить статус +systemctl status mitmproxy + +# Логи +journalctl -u mitmproxy -f +``` + +--- + +## 5. Настройка телефона + +### Android + +1. Настройки → Wi-Fi +2. Долгое нажатие на сеть → Modify network +3. Advanced → Proxy: Manual +4. Proxy hostname: `` +5. Proxy port: `8080` +6. Сохранить + +### iPhone + +1. Настройки → Wi-Fi +2. Нажать на ⓘ рядом с сетью +3. Configure Proxy: Manual +4. Server: `` +5. Port: `8080` +6. Authentication: ON + - Username: `claw` + - Password: `KworkParser2026!` +7. Save + +### Установка сертификата (Android) + +1. Открыть браузер на телефоне +2. Перейти: `http://:8080/mitmproxy/mitmproxy-ca.cer` +3. Скачать сертификат +4. Настройки → Security → Install certificate +5. Выбрать файл: `mitmproxy-ca.cer` +6. Name: `mitmproxy` +7. OK + +### Установка сертификата (iPhone) + +1. Safari: `http://:8080/mitmproxy/mitmproxy-ca.cer` +2. Allow → Profile downloaded +3. Настройки → Profile downloaded → Install +4. Настройки → General → About → Certificate Trust Settings +5. Включить доверие для mitmproxy + +--- + +## 6. Проверка работы + +### С телефона + +```bash +# Открыть браузер на телефоне +# Перейти на http://example.com +``` + +### На VPS + +```bash +# Посмотреть логи +tail -f /root/kwork-parser/mitmproxy/mitmproxy.log + +# Или через скрипт +python3 /root/kwork-parser/scripts/view_flows.py --last 10 +``` + +--- + +## 7. Просмотр логов + +### Последние запросы + +```bash +python3 /root/kwork-parser/scripts/view_flows.py --last 20 +``` + +### Только Kwork + +```bash +python3 /root/kwork-parser/scripts/view_flows.py --domain kwork.ru --last 50 +``` + +### Только JSON + +```bash +python3 /root/kwork-parser/scripts/view_flows.py --json --last 20 +``` + +### Полная информация + +```bash +python3 /root/kwork-parser/scripts/view_flows.py --domain kwork.ru --full +``` + +### В реальном времени + +```bash +tail -f /root/kwork-parser/mitmproxy/flows.mitm | \ + python3 -c " +import sys +from mitmproxy import io +for flow in io.FlowReader(sys.stdin.buffer).read(): + if hasattr(flow, 'request'): + print(f'{flow.request.method} {flow.request.url}') +" +``` + +--- + +## 8. Безопасность + +### Что защищено + +✅ **Авторизация** — Basic Auth (логин/пароль) +✅ **Firewall** — только порт 8080 открыт +✅ **Изоляция** — mitmproxy работает от root (для systemd) + +### Рекомендации + +1. **Смени пароль** после первого запуска: + ```bash + # В auth.py и start.sh + auth_pass="НовыйПароль2026!" + ``` + +2. **Ограничь IP** (если возможно): + ```bash + # В start.sh добавить: + --set allow_hosts=["твой_домашний_IP"] + ``` + +3. **Используй HTTPS** для mitmweb: + ```bash + mitmweb --web-port 8081 --ssl-keyfile key.pem --ssl-certfile cert.pem + ``` + +4. **Очищай логи** регулярно: + ```bash + # Cron: очищать раз в день + 0 0 * * * truncate -s 0 /root/kwork-parser/mitmproxy/flows.mitm + ``` + +5. **Не логируй чувствительные данные**: + ```bash + # Игнорировать banking сайты + mitmdump --set 'ignore_hosts=~u (bank|payment|stripe)"' + ``` + +--- + +## 9. Troubleshooting + +### Ошибка: Address already in use + +```bash +# Проверить кто использует порт +lsof -i :8080 + +# Убить процесс +kill -9 +``` + +### Ошибка: Certificate not trusted + +```bash +# Переустановить сертификат на телефоне +# Удалить старый из доверенных +# Скачать заново: http://:8080/mitmproxy/mitmproxy-ca.cer +``` + +### Ошибка: Authentication failed + +```bash +# Проверить логин/пароль +# Логин: claw +# Пароль: KworkParser2026! + +# В настройках прокси телефона включить Authentication +``` + +### mitmproxy не стартует + +```bash +# Проверить логи +journalctl -u mitmproxy -n 50 + +# Проверить конфиг +cat /root/kwork-parser/mitmproxy/mitmproxy.log +``` + +--- + +## 10. Команды для Claw (AI) + +```bash +# Посмотреть последние запросы +python3 /root/kwork-parser/scripts/view_flows.py --last 20 + +# Найти API эндпоинты +python3 /root/kwork-parser/scripts/view_flows.py --json --domain kwork.ru + +# Экспортировать в HAR +python3 /root/kwork-parser/scripts/mitmproxy-to-har.py \ + /root/kwork-parser/mitmproxy/flows.mitm \ + /root/kwork-parser/mitmproxy/export.har + +# Анализ HAR +jq '[.log.entries[] | select(.request.url | contains("kwork"))] | length' \ + /root/kwork-parser/mitmproxy/export.har +``` + +--- + +_Updated: 2026-03-22_ diff --git a/mitmproxy/auth.py b/mitmproxy/auth.py new file mode 100644 index 0000000..6b59569 --- /dev/null +++ b/mitmproxy/auth.py @@ -0,0 +1,46 @@ +#!/usr/bin/env python3 +""" +mitmproxy addon: HTTP Basic Auth +Usage: mitmdump -s auth.py --set auth_user=admin --set auth_pass=secret +""" + +from mitmproxy import http, ctx + + +class HTTPAuth: + def __init__(self): + self.auth_user = ctx.options.auth_user + self.auth_pass = ctx.options.auth_pass + + def request(self, flow: http.HTTPFlow): + # Skip auth for mitm.it (certificate download) + if "mitm.it" in flow.request.pretty_host: + return + + # Check for Authorization header + auth_header = flow.request.headers.get("Authorization", "") + + if not auth_header.startswith("Basic "): + self.challenge_auth(flow) + return + + # Decode and verify credentials + import base64 + try: + credentials = base64.b64decode(auth_header[6:]).decode() + username, password = credentials.split(":", 1) + + if username != self.auth_user or password != self.auth_pass: + self.challenge_auth(flow) + except Exception: + self.challenge_auth(flow) + + def challenge_auth(self, flow: http.HTTPFlow): + flow.response = http.Response.make( + 407, + b"Proxy Authentication Required", + {"Proxy-Authenticate": 'Basic realm="mitmproxy"'} + ) + + +addons = [HTTPAuth()] diff --git a/mitmproxy/mitmproxy.conf b/mitmproxy/mitmproxy.conf new file mode 100644 index 0000000..ec88ad1 --- /dev/null +++ b/mitmproxy/mitmproxy.conf @@ -0,0 +1,34 @@ +# mitmproxy configuration + +# Authentication +auth_user = "claw" +auth_pass = "KworkParser2026!" + +# Network +mode = "regular" +listen_port = 8080 +listen_host = "0.0.0.0" + +# SSL +ssl_insecure = false +add_upstream_certs_to_client_chain = true + +# Logging +verbose = true +termlog_verbosity = "info" + +# Save flows +save_stream_file = "/root/kwork-parser/mitmproxy/flows.mitm" + +# Ignore common CDNs and analytics (reduce noise) +ignore_hosts = [ + "~u cdn.*\\.google\\.com", + "~u .*\\.google-analytics\\.com", + "~u .*\\.yandex\\.ru", + "~u .*\\.yandex\\.net", + "~u .*\\.facebook\\.com", +] + +# Keep only Kwork and related domains +# Uncomment to filter strictly: +# filter = "~u (kwork\\.ru|kworks\\.ru|kwork-api\\.ru)" diff --git a/mitmproxy/start.sh b/mitmproxy/start.sh new file mode 100755 index 0000000..1725a46 --- /dev/null +++ b/mitmproxy/start.sh @@ -0,0 +1,72 @@ +#!/bin/bash +# Start mitmproxy with authentication + +set -e + +CONFIG_DIR="/root/kwork-parser/mitmproxy" +LOG_FILE="/root/kwork-parser/mitmproxy/mitmproxy.log" +PID_FILE="/root/kwork-parser/mitmproxy/mitmproxy.pid" + +echo "=== Mitmproxy Startup ===" +echo "Config: $CONFIG_DIR/mitmproxy.conf" +echo "Auth addon: $CONFIG_DIR/auth.py" +echo "Log: $LOG_FILE" +echo "" + +# Check if already running +if [ -f "$PID_FILE" ]; then + OLD_PID=$(cat "$PID_FILE") + if kill -0 "$OLD_PID" 2>/dev/null; then + echo "⚠️ mitmproxy already running (PID: $OLD_PID)" + echo "Stop with: sudo systemctl stop mitmproxy" + exit 1 + fi +fi + +# Create certificate if not exists +if [ ! -f ~/.mitmproxy/mitmproxy-ca.pem ]; then + echo "📜 Generating SSL certificate..." + mitmproxy --version > /dev/null +fi + +# Start mitmproxy +echo "🚀 Starting mitmproxy..." +nohup mitmdump \ + -s "$CONFIG_DIR/auth.py" \ + --set auth_user=claw \ + --set auth_pass="KworkParser2026!" \ + -p 8080 \ + --mode regular \ + --set ssl_insecure=false \ + --set add_upstream_certs_to_client_chain=true \ + -w "$CONFIG_DIR/flows.mitm" \ + -v \ + > "$LOG_FILE" 2>&1 & + +echo $! > "$PID_FILE" +sleep 2 + +# Check if running +if kill -0 $(cat "$PID_FILE") 2>/dev/null; then + echo "✅ mitmproxy started successfully!" + echo "" + echo "📱 Phone configuration:" + echo " Proxy: $(curl -s ifconfig.me):8080" + echo " Login: claw" + echo " Password: KworkParser2026!" + echo "" + echo "🔒 Certificate:" + echo " http://$(curl -s ifconfig.me):8080/mitmproxy/mitmproxy-ca.cer" + echo "" + echo "📊 View logs:" + echo " tail -f $LOG_FILE" + echo " python3 /root/kwork-parser/scripts/view_flows.py" + echo "" + echo "🛑 Stop:" + echo " sudo systemctl stop mitmproxy" + echo " or: kill $(cat $PID_FILE)" +else + echo "❌ Failed to start mitmproxy" + echo "Check log: $LOG_FILE" + exit 1 +fi diff --git a/scripts/mitmproxy-to-har.py b/scripts/mitmproxy-to-har.py old mode 100644 new mode 100755 diff --git a/scripts/view_flows.py b/scripts/view_flows.py new file mode 100755 index 0000000..bb9d54b --- /dev/null +++ b/scripts/view_flows.py @@ -0,0 +1,106 @@ +#!/usr/bin/env python3 +""" +View mitmproxy flows from log file +Usage: python3 view_flows.py [options] + +Options: + --last N Show last N requests (default: 20) + --domain X Filter by domain + --method X Filter by method (GET/POST) + --json Show only JSON responses + --full Show full request/response +""" + +import sys +import json +from pathlib import Path +from mitmproxy import io + + +FLOWS_FILE = "/root/kwork-parser/mitmproxy/flows.mitm" + + +def view_flows(last_n=20, domain_filter=None, method_filter=None, json_only=False, full=False): + """View flows from mitmproxy dump.""" + + if not Path(FLOWS_FILE).exists(): + print(f"❌ Flows file not found: {FLOWS_FILE}") + print("Make sure mitmproxy is running and capturing traffic.") + return + + flows = [] + + with open(FLOWS_FILE, 'rb') as f: + reader = io.FlowReader(f) + for flow in reader.read(): + from mitmproxy.http import HTTPFlow + if isinstance(flow, HTTPFlow) and flow.request and flow.response: + flows.append(flow) + + # Filter + if domain_filter: + flows = [f for f in flows if domain_filter in f.request.url] + + if method_filter: + flows = [f for f in flows if f.request.method == method_filter] + + if json_only: + flows = [f for f in flows if f.response.headers.get("Content-Type", "").startswith("application/json")] + + # Sort by time (newest first) + flows.sort(key=lambda x: x.request.timestamp_start, reverse=True) + + # Limit + flows = flows[:last_n] + + # Display + print(f"📊 Showing {len(flows)} flows (newest first)\n") + + for i, flow in enumerate(flows, 1): + method = flow.request.method + url = flow.request.url + status = flow.response.status_code + size = len(flow.response.content) if flow.response.content else 0 + content_type = flow.response.headers.get("Content-Type", "unknown").split(";")[0] + time_ms = int((flow.response.timestamp_end - flow.request.timestamp_start) * 1000) if flow.response.timestamp_end else 0 + + # Color coding + status_color = "🟢" if status < 400 else "🟡" if status < 500 else "🔴" + + print(f"{i}. {status_color} {method} {url[:80]}") + print(f" Status: {status} | Size: {size:,} bytes | Time: {time_ms}ms | Type: {content_type}") + + if full: + print(f"\n Request Headers:") + for k, v in flow.request.headers.items(): + print(f" {k}: {v[:100]}") + + if flow.request.method == "POST" and flow.request.content: + print(f"\n Request Body:") + print(f" {flow.request.get_text()[:500]}") + + print(f"\n Response:") + print(f" {flow.response.get_text()[:500]}") + + print() + + +if __name__ == "__main__": + import argparse + + parser = argparse.ArgumentParser(description="View mitmproxy flows") + parser.add_argument("--last", type=int, default=20, help="Show last N requests") + parser.add_argument("--domain", type=str, help="Filter by domain") + parser.add_argument("--method", type=str, help="Filter by method") + parser.add_argument("--json", action="store_true", help="Show only JSON responses") + parser.add_argument("--full", action="store_true", help="Show full request/response") + + args = parser.parse_args() + + view_flows( + last_n=args.last, + domain_filter=args.domain, + method_filter=args.method, + json_only=args.json, + full=args.full + )